1. 数据库事务基础
1.1 ACID特性
01.ACID四大特性概述
a.原子性Atomicity
a.概念定义
事务中的所有操作作为一个不可分割的工作单元，要么全部成功执行，要么全部失败回滚到事务开始前的状态。这是通过数据库的回滚日志实现的。
b.实现原理
数据库通过Undo Log记录事务执行前的数据状态，当事务需要回滚时，系统会读取Undo Log中的记录，将数据恢复到事务开始前的状态，确保事务的原子性。
c.代码示例
---
# 原子性示例：银行转账事务
import sqlite3
def atomic_transfer_demo():
"""演示事务的原子性特性"""
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建账户表
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
balance REAL NOT NULL
)
''')
# 插入测试数据
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Alice', 1000.0))
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Bob', 500.0))
conn.commit()
# 开始转账事务
try:
cursor.execute('BEGIN TRANSACTION')
# 从Alice账户扣除100元
cursor.execute('UPDATE accounts SET balance = balance - ? WHERE name = ?', (100, 'Alice'))
# 模拟系统错误（这里故意抛出异常）
if True: # 模拟异常发生
raise Exception("系统异常，转账失败")
# 向Bob账户增加100元
cursor.execute('UPDATE accounts SET balance = balance + ? WHERE name = ?', (100, 'Bob'))
conn.commit()
print("转账成功")
except Exception as e:
print(f"转账失败: {e}")
conn.rollback() # 回滚事务，Alice的100元被恢复
print("事务已回滚")
# 检查最终余额
cursor.execute('SELECT name, balance FROM accounts')
accounts = cursor.fetchall()
for account in accounts:
print(f"{account[0]}: {account[1]}")
conn.close()
if __name__ == "__main__":
atomic_transfer_demo()
---
b.一致性Consistency
a.概念定义
事务必须使数据库从一个有效的状态转变到另一个有效的状态，它维护了数据库的完整性约束。事务执行的结果必须是使数据库从一个一致性状态变到另一个一致性状态。
b.约束验证
数据库在执行每个事务操作前后都会检查完整性约束，包括主键约束、外键约束、唯一性约束、检查约束等。如果违反约束，事务将被回滚。
c.代码示例
---
# 一致性示例：维护账户余额不能为负的约束
def consistency_demo():
"""演示事务的一致性特性"""
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建带有约束的账户表
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
# 插入测试数据
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Alice', 100.0))
conn.commit()
# 尝试执行违反约束的操作
try:
cursor.execute('BEGIN TRANSACTION')
# 尝试提取超过余额的金额
cursor.execute('UPDATE accounts SET balance = balance - ? WHERE name = ?', (150, 'Alice'))
# 这将违反CHECK约束
conn.commit()
except sqlite3.IntegrityError as e:
print(f"一致性约束违反: {e}")
conn.rollback()
print("事务已回滚，保持数据一致性")
# 验证数据状态
cursor.execute('SELECT name, balance FROM accounts')
result = cursor.fetchone()
print(f"最终余额: {result[1]}") # 应该仍然是100
conn.close()
if __name__ == "__main__":
consistency_demo()
---
c.隔离性Isolation
a.概念定义
多个并发事务的执行之间互不干扰，每个事务感觉不到其他事务的存在。数据库系统提供了不同的隔离级别来控制并发事务之间的可见性。
b.隔离级别
包括读未提交(Read Uncommitted)、读已提交(Read Committed)、可重复读(Repeatable Read)和串行化(Serializable)四个级别，隔离级别越高，并发性能越低，但数据一致性越强。
c.代码示例
---
# 隔离性示例：并发事务隔离
import threading
import time
import sqlite3
class IsolationDemo:
def __init__(self):
self.conn = sqlite3.connect(':memory:', check_same_thread=False)
self.cursor = self.conn.cursor()
self.setup_database()
def setup_database(self):
"""初始化数据库"""
self.cursor.execute('''
CREATE TABLE counter (
id INTEGER PRIMARY KEY,
value INTEGER NOT NULL
)
''')
self.cursor.execute('INSERT INTO counter (id, value) VALUES (1, 0)')
self.conn.commit()
def transaction1(self):
"""事务1：读取-修改-写入"""
try:
self.cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 读取当前值
self.cursor.execute('SELECT value FROM counter WHERE id = 1')
current_value = self.cursor.fetchone()[0]
print(f"事务1读取到值: {current_value}")
# 模拟长时间处理
time.sleep(2)
# 增加值并写回
new_value = current_value + 10
self.cursor.execute('UPDATE counter SET value = ? WHERE id = 1', (new_value,))
print(f"事务1写入新值: {new_value}")
self.conn.commit()
print("事务1提交成功")
except Exception as e:
print(f"事务1失败: {e}")
self.conn.rollback()
def transaction2(self):
"""事务2：读取-修改-写入"""
time.sleep(0.5) # 稍后开始
try:
# 使用独立的连接来模拟真正的并发
conn2 = sqlite3.connect(':memory:')
cursor2 = conn2.cursor()
cursor2.execute('BEGIN IMMEDIATE TRANSACTION')
# 这里会等待事务1释放锁
cursor2.execute('SELECT value FROM counter WHERE id = 1')
current_value = cursor2.fetchone()[0]
print(f"事务2读取到值: {current_value}")
new_value = current_value + 5
cursor2.execute('UPDATE counter SET value = ? WHERE id = 1', (new_value,))
print(f"事务2写入新值: {new_value}")
conn2.commit()
print("事务2提交成功")
conn2.close()
except Exception as e:
print(f"事务2失败: {e}")
if __name__ == "__main__":
demo = IsolationDemo()
# 启动两个并发事务
t1 = threading.Thread(target=demo.transaction1)
t2 = threading.Thread(target=demo.transaction2)
t1.start()
t2.start()
t1.join()
t2.join()
# 检查最终结果
demo.cursor.execute('SELECT value FROM counter WHERE id = 1')
final_value = demo.cursor.fetchone()[0]
print(f"最终值: {final_value}")
---
d.持久性Durability
a.概念定义
一旦事务提交，其结果就是永久性的，即使系统发生故障也不会丢失。这是通过数据库的预写日志(WAL)和重做日志(Redo Log)机制实现的。
b.持久化机制
数据库在提交事务时，会先将相关的日志记录写入到持久存储设备中，确保即使系统崩溃，也能通过重放日志来恢复已提交的事务。
c.代码示例
---
# 持久性示例：WAL模式和强制刷盘
def durability_demo():
"""演示事务的持久性特性"""
# 使用WAL模式提高性能和可靠性
conn = sqlite3.connect('durability_demo.db')
cursor = conn.cursor()
# 启用WAL模式和外键约束
cursor.execute('PRAGMA journal_mode=WAL')
cursor.execute('PRAGMA foreign_keys=ON')
# 创建重要数据表
cursor.execute('''
CREATE TABLE important_data (
id INTEGER PRIMARY KEY AUTOINCREMENT,
message TEXT NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 模拟重要的事务操作
critical_messages = [
"系统启动",
"用户登录成功",
"关键数据更新",
"业务流程完成"
]
try:
for i, message in enumerate(critical_messages):
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 插入关键数据
cursor.execute('INSERT INTO important_data (message) VALUES (?)', (message,))
# 模拟一些处理逻辑
if i == 2:
# 模拟在第三个操作时的特殊处理
print(f"处理关键数据: {message}")
# 强制同步到磁盘确保持久性
cursor.execute('PRAGMA synchronous = FULL')
conn.commit()
print(f"已持久化: {message}")
except Exception as e:
print(f"事务失败: {e}")
conn.rollback()
finally:
conn.close()
# 验证数据持久性
print("\n验证数据持久性:")
conn_verify = sqlite3.connect('durability_demo.db')
cursor_verify = conn_verify.cursor()
cursor_verify.execute('SELECT message, timestamp FROM important_data ORDER BY id')
for row in cursor_verify.fetchall():
print(f" {row[1]}: {row[0]}")
conn_verify.close()
# 清理演示文件
import os
os.remove('durability_demo.db')
if __name__ == "__main__":
durability_demo()
---
02.ACID特性的重要性
a.数据完整性保证
ACID特性确保数据库在各种异常情况下都能保持数据的完整性和一致性，防止出现数据不一致或丢失的情况，是企业级应用的核心要求。
b.并发控制基础
ACID特性为多用户并发访问提供了可靠的控制机制，确保并发事务之间不会相互干扰，维护了多用户环境下的数据一致性。
c.故障恢复能力
通过持久性和原子性特性，系统能够在发生故障时自动恢复到一致的状态，保证了系统的可靠性和可用性。
1.2 事务的开始与提交
01.事务生命周期管理
a.事务开始
a.显式开始
使用BEGIN、START TRANSACTION或BEGIN TRANSACTION语句明确启动事务，这是最常见的事务开始方式，确保后续操作都在事务控制下执行。
b.隐式开始
在禁用自动提交模式时，第一个SQL语句会自动开始一个新事务，这种方式简化了操作但可能降低代码可读性。
c.代码示例
---
# 事务开始方式对比
import sqlite3
def transaction_start_demo():
"""演示不同的事务开始方式"""
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE employees (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
salary REAL NOT NULL
)
''')
conn.commit()
# 方式1：显式BEGIN语句
print("方式1：使用BEGIN语句")
cursor.execute('BEGIN TRANSACTION')
cursor.execute('INSERT INTO employees (name, salary) VALUES (?, ?)', ('Alice', 5000))
cursor.execute('INSERT INTO employees (name, salary) VALUES (?, ?)', ('Bob', 6000))
conn.commit()
print(" 使用BEGIN的事务已提交")
# 方式2：直接操作（自动开始）
print("方式2：直接操作（禁用自动提交后）")
# 注意：sqlite3的Python接口默认自动提交为False
cursor.execute('INSERT INTO employees (name, salary) VALUES (?, ?)', ('Charlie', 5500))
conn.commit()
print(" 隐式开始的事务已提交")
# 方式3：使用事务装饰器（后续章节详细介绍）
print("查看结果：")
cursor.execute('SELECT * FROM employees')
employees = cursor.fetchall()
for emp in employees:
print(f" ID:{emp[0]}, 姓名:{emp[1]}, 薪资:{emp[2]}")
conn.close()
if __name__ == "__main__":
transaction_start_demo()
---
b.事务提交
a.提交机制
使用COMMIT语句将事务中的所有更改永久保存到数据库，这个过程包括将变更写入数据文件和日志文件，确保持久性。
b.提交验证
提交过程中数据库会进行完整性检查，确保所有操作都符合约束条件，如果有违反约束的操作，整个事务将被回滚。
c.代码示例
---
# 事务提交的完整流程
def commit_process_demo():
"""演示事务提交的完整流程"""
conn = sqlite3.connect('commit_demo.db')
cursor = conn.cursor()
# 启用外键约束和WAL模式
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
# 创建相关表
cursor.execute('''
CREATE TABLE departments (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL UNIQUE
)
''')
cursor.execute('''
CREATE TABLE employees (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
dept_id INTEGER,
salary REAL NOT NULL,
FOREIGN KEY (dept_id) REFERENCES departments(id)
)
''')
conn.commit()
try:
# 开始事务
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print("事务开始")
# 插入部门数据
cursor.execute('INSERT INTO departments (name) VALUES (?)', ('技术部',))
dept_id = cursor.lastrowid
print(f"插入技术部，ID: {dept_id}")
# 插入员工数据
employees_data = [
('张三', dept_id, 8000),
('李四', dept_id, 7500),
('王五', dept_id, 9000)
]
for name, dept, salary in employees_data:
cursor.execute('INSERT INTO employees (name, dept_id, salary) VALUES (?, ?, ?)',
(name, dept, salary))
print(f"插入员工: {name}, 薪资: {salary}")
# 模拟数据验证
cursor.execute('SELECT COUNT(*) FROM employees WHERE dept_id = ?', (dept_id,))
emp_count = cursor.fetchone()[0]
print(f"技术部员工总数: {emp_count}")
# 检查薪资总额（业务规则验证）
cursor.execute('SELECT SUM(salary) FROM employees WHERE dept_id = ?', (dept_id,))
total_salary = cursor.fetchone()[0]
print(f"技术部薪资总额: {total_salary}")
# 如果一切正常，提交事务
conn.commit()
print("✅ 事务提交成功！所有更改已保存")
except Exception as e:
print(f"❌ 事务提交失败: {e}")
conn.rollback()
print("事务已回滚，所有更改被撤销")
finally:
# 验证最终结果
cursor.execute('SELECT d.name, COUNT(e.id) as emp_count, SUM(e.salary) as total_salary FROM departments d LEFT JOIN employees e ON d.id = e.dept_id GROUP BY d.id')
results = cursor.fetchall()
print("\n最终数据状态:")
for dept_name, emp_count, total_salary in results:
print(f" 部门: {dept_name}, 员工数: {emp_count}, 薪资总额: {total_salary}")
conn.close()
# 清理演示文件
import os
if os.path.exists('commit_demo.db'):
os.remove('commit_demo.db')
if __name__ == "__main__":
commit_process_demo()
---
c.提交模式选择
a.立即提交IMMEDIATE
BEGIN IMMEDIATE会立即获取保留锁，防止其他事务写入，提供了更好的并发控制策略，适用于写操作较多的场景。
b.排他提交EXCLUSIVE
BEGIN EXCLUSIVE获取排他锁，阻止其他事务的读写操作，提供了最严格的事务隔离，但会严重影响并发性能。
c.延迟提交DEFERRED
BEGIN DEFERRED是默认模式，不立即获取锁，直到第一次读写操作时才获取相应锁，提供了最好的并发性能。
02.提交策略与性能优化
a.批量提交优化
将多个相关操作组合在一个事务中提交，减少磁盘I/O次数和网络开销，显著提高批量操作的性能。
b.延迟写入策略
通过WAL模式实现延迟写入，将变更先写入内存缓冲区，定期批量刷盘，平衡了性能和持久性要求。
c.代码示例
---
# 批量提交与性能优化
import time
import sqlite3
def batch_commit_performance():
"""演示批量提交的性能优势"""
conn = sqlite3.connect('performance_test.db')
cursor = conn.cursor()
# 启用WAL模式提高性能
cursor.execute('PRAGMA journal_mode = WAL')
cursor.execute('PRAGMA synchronous = NORMAL')
# 创建测试表
cursor.execute('''
CREATE TABLE test_data (
id INTEGER PRIMARY KEY,
data TEXT NOT NULL,
value REAL NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
conn.commit()
# 准备测试数据
test_data = [
(f'data_{i}', i * 1.5) for i in range(1, 10001)
]
# 测试1：逐条提交（性能差）
print("测试1：逐条提交模式")
start_time = time.time()
for i, (data, value) in enumerate(test_data[:1000]): # 只测试1000条
cursor.execute('BEGIN TRANSACTION')
cursor.execute('INSERT INTO test_data (data, value) VALUES (?, ?)', (data, value))
conn.commit()
single_commit_time = time.time() - start_time
print(f" 1000条记录逐条提交耗时: {single_commit_time:.4f}秒")
# 清空数据
cursor.execute('DELETE FROM test_data')
conn.commit()
# 测试2：批量提交（性能好）
print("\n测试2：批量提交模式")
start_time = time.time()
cursor.execute('BEGIN TRANSACTION')
for i, (data, value) in enumerate(test_data):
cursor.execute('INSERT INTO test_data (data, value) VALUES (?, ?)', (data, value))
# 每1000条提交一次
if (i + 1) % 1000 == 0:
conn.commit()
print(f" 已提交 {i + 1} 条记录")
if i + 1 < len(test_data):
cursor.execute('BEGIN TRANSACTION')
# 提交剩余的记录
conn.commit()
batch_commit_time = time.time() - start_time
print(f"10000条记录批量提交耗时: {batch_commit_time:.4f}秒")
# 性能对比
improvement = (single_commit_time * 10) / batch_commit_time # 假设逐条提交10000条的时间
print(f"\n性能提升: {improvement:.2f}倍")
# 验证数据完整性
cursor.execute('SELECT COUNT(*) FROM test_data')
total_records = cursor.fetchone()[0]
print(f"总共插入记录数: {total_records}")
conn.close()
# 清理测试文件
import os
if os.path.exists('performance_test.db'):
os.remove('performance_test.db')
if __name__ == "__main__":
batch_commit_performance()
---
03.事务提交的最佳实践
a.事务粒度控制
保持事务尽可能简短，只在必要时包含操作，避免长时间占用锁资源，提高系统并发能力。
b.异常处理机制
实现完善的异常捕获和错误处理，确保在任何情况下都能正确提交或回滚事务，维护数据一致性。
c.性能监控与调优
监控事务执行时间和资源使用情况，根据业务特点调整事务策略，找到性能与一致性的最佳平衡点。
1.3 事务回滚
01.回滚机制原理
a.回滚触发条件
a.显式回滚
当应用程序检测到业务逻辑错误或数据验证失败时，主动调用ROLLBACK语句撤销事务中的所有操作，确保数据状态的一致性。
b.异常自动回滚
当系统发生严重错误、死锁、或违反数据库约束时，数据库引擎会自动执行回滚操作，防止数据损坏。
c.连接中断回滚
当数据库连接意外中断或客户端崩溃时，数据库会自动回滚未完成的事务，保证数据完整性。
b.回滚执行过程
a.读取Undo日志
系统读取事务执行过程中记录的Undo日志，获取每个操作前的数据状态，为回滚操作提供必要信息。
b.逆向操作执行
按照与事务执行相反的顺序，逐一撤销每个操作，将数据恢复到事务开始前的状态。
c.资源释放
释放事务占用的所有锁资源，清除临时数据结构，确保系统资源得到及时回收。
c.代码示例
---
# 事务回滚的完整演示
import sqlite3
import sys
def rollback_mechanism_demo():
"""演示事务回滚的各种触发条件"""
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
cursor.execute('''
CREATE TABLE transaction_log (
id INTEGER PRIMARY KEY,
account_id INTEGER,
amount REAL NOT NULL,
operation TEXT NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (account_id) REFERENCES accounts(id)
)
''')
# 插入初始数据
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Alice', 1000.0))
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Bob', 500.0))
conn.commit()
print("=== 场景1：业务逻辑错误导致的显式回滚 ===")
try:
cursor.execute('BEGIN TRANSACTION')
print("事务开始")
# Alice向Bob转账200元
cursor.execute('UPDATE accounts SET balance = balance - ? WHERE name = ?', (200, 'Alice'))
print("Alice账户扣除200元")
# 检查Alice余额是否足够（业务规则）
cursor.execute('SELECT balance FROM accounts WHERE name = ?', ('Alice',))
alice_balance = cursor.fetchone()[0]
print(f"Alice当前余额: {alice_balance}")
if alice_balance < 100:
# 业务规则：余额不能低于100元
raise ValueError("账户余额低于最低限额，交易被拒绝")
# 继续转账操作
cursor.execute('UPDATE accounts SET balance = balance + ? WHERE name = ?', (200, 'Bob'))
print("Bob账户增加200元")
# 记录交易日志
cursor.execute('''
INSERT INTO transaction_log (account_id, amount, operation)
SELECT id, ?, ? FROM accounts WHERE name = ?
''', (200, 'transfer_out', 'Alice'))
cursor.execute('''
INSERT INTO transaction_log (account_id, amount, operation)
SELECT id, ?, ? FROM accounts WHERE name = ?
''', (200, 'transfer_in', 'Bob'))
# 如果一切正常，提交事务
conn.commit()
print("✅ 转账成功，事务已提交")
except ValueError as e:
print(f"❌ 业务错误: {e}")
conn.rollback()
print("事务已回滚，所有操作被撤销")
# 验证回滚结果
cursor.execute('SELECT name, balance FROM accounts')
accounts = cursor.fetchall()
print("账户状态:")
for name, balance in accounts:
print(f" {name}: {balance}")
print("\n=== 场景2：约束违反导致的自动回滚 ===")
try:
cursor.execute('BEGIN TRANSACTION')
# 尝试提取超过Bob余额的金额
cursor.execute('UPDATE accounts SET balance = balance - ? WHERE name = ?', (600, 'Bob'))
print("Bob账户扣除600元")
# 这个操作会违反CHECK约束
conn.commit()
print("事务提交成功")
except sqlite3.IntegrityError as e:
print(f"❌ 约束违反: {e}")
print("数据库自动回滚事务")
# 验证自动回滚结果
cursor.execute('SELECT name, balance FROM accounts')
accounts = cursor.fetchall()
print("账户状态:")
for name, balance in accounts:
print(f" {name}: {balance}")
print("\n=== 场景3：系统异常导致的回滚 ===")
try:
cursor.execute('BEGIN TRANSACTION')
# 执行一系列复杂操作
cursor.execute('UPDATE accounts SET balance = balance + 100 WHERE name = ?', ('Alice',))
print("Alice账户增加100元")
# 模拟系统异常
raise RuntimeError("模拟系统崩溃")
cursor.execute('UPDATE accounts SET balance = balance - 100 WHERE name = ?', ('Bob',))
conn.commit()
except Exception as e:
print(f"❌ 系统异常: {e}")
conn.rollback()
print("异常处理：事务已回滚")
# 最终验证
cursor.execute('SELECT name, balance FROM accounts')
accounts = cursor.fetchall()
print("最终账户状态:")
for name, balance in accounts:
print(f" {name}: {balance}")
conn.close()
if __name__ == "__main__":
rollback_mechanism_demo()
---
02.回滚策略与最佳实践
a.回滚点设置
a.保存点概念
在长事务中设置多个保存点，允许部分回滚而不是完全回滚，提高了复杂业务流程的灵活性和性能。
b.嵌套回滚处理
通过多层保存点实现嵌套回滚，支持复杂的业务逻辑分支和错误恢复策略。
c.代码示例
---
# 保存点与部分回滚演示
def savepoint_demo():
"""演示保存点和部分回滚的使用"""
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建订单相关表
cursor.execute('''
CREATE TABLE orders (
id INTEGER PRIMARY KEY,
customer_name TEXT NOT NULL,
total_amount REAL NOT NULL,
status TEXT NOT NULL DEFAULT 'pending'
)
''')
cursor.execute('''
CREATE TABLE order_items (
id INTEGER PRIMARY KEY,
order_id INTEGER,
product_name TEXT NOT NULL,
quantity INTEGER NOT NULL,
price REAL NOT NULL,
FOREIGN KEY (order_id) REFERENCES orders(id)
)
''')
cursor.execute('''
CREATE TABLE inventory (
id INTEGER PRIMARY KEY,
product_name TEXT NOT NULL UNIQUE,
stock_quantity INTEGER NOT NULL
)
''')
# 初始化库存
cursor.executemany('INSERT INTO inventory (product_name, stock_quantity) VALUES (?, ?)', [
('笔记本电脑', 10),
('无线鼠标', 20),
('机械键盘', 15),
('显示器', 8)
])
conn.commit()
try:
cursor.execute('BEGIN TRANSACTION')
print("开始创建订单事务")
# 步骤1：创建订单
cursor.execute('''
INSERT INTO orders (customer_name, total_amount)
VALUES (?, ?)
''', ('张三', 0)) # 先创建订单，稍后计算总价
order_id = cursor.lastrowid
print(f"创建订单，ID: {order_id}")
# 设置第一个保存点
cursor.execute('SAVEPOINT save_order_created')
print("设置保存点: save_order_created")
# 步骤2：添加订单项并扣减库存
order_items = [
('笔记本电脑', 1, 5999.0),
('无线鼠标', 2, 199.0),
('机械键盘', 1, 599.0)
]
total_amount = 0
for product, quantity, price in order_items:
# 检查库存
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"产品 {product} 库存不足，当前库存: {stock}, 需要: {quantity}")
# 扣减库存
cursor.execute('''
UPDATE inventory SET stock_quantity = stock_quantity - ?
WHERE product_name = ?
''', (quantity, product))
print(f"扣减库存: {product} x{quantity}")
# 添加订单项
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, price)
VALUES (?, ?, ?, ?)
''', (order_id, product, quantity, price))
total_amount += quantity * price
# 设置第二个保存点
cursor.execute('SAVEPOINT save_inventory_updated')
print("设置保存点: save_inventory_updated")
# 步骤3：尝试应用折扣（可能失败）
try:
discount = 0.1 # 10%折扣
if total_amount > 5000:
total_amount *= (1 - discount)
print(f"应用10%折扣，折扣后金额: {total_amount:.2f}")
# 更新订单总金额
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (total_amount, order_id))
print("订单状态更新为已确认")
except Exception as discount_error:
print(f"折扣应用失败: {discount_error}")
print("回滚到保存点: save_inventory_updated")
cursor.execute('ROLLBACK TO SAVEPOINT save_inventory_updated')
# 不应用折扣，继续处理
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (total_amount, order_id))
print("不应用折扣，订单已确认")
# 步骤4：最终验证
cursor.execute('SELECT COUNT(*) FROM order_items WHERE order_id = ?', (order_id,))
item_count = cursor.fetchone()[0]
if item_count == 0:
print("没有订单项，回滚到开始")
cursor.execute('ROLLBACK TO SAVEPOINT save_order_created')
raise ValueError("订单必须包含至少一个商品")
# 提交整个事务
conn.commit()
print("✅ 订单创建成功并已提交")
# 显示订单详情
cursor.execute('''
SELECT o.customer_name, o.total_amount, o.status,
COUNT(oi.id) as item_count
FROM orders o
LEFT JOIN order_items oi ON o.id = oi.order_id
WHERE o.id = ?
GROUP BY o.id
''', (order_id,))
order_info = cursor.fetchone()
print(f"订单详情: 客户={order_info[0]}, 总额={order_info[1]}, 状态={order_info[2]}, 商品数={order_info[3]}")
# 显示剩余库存
cursor.execute('SELECT product_name, stock_quantity FROM inventory')
print("剩余库存:")
for product, stock in cursor.fetchall():
print(f" {product}: {stock}")
except Exception as e:
print(f"❌ 订单创建失败: {e}")
conn.rollback()
print("整个事务已回滚")
finally:
conn.close()
if __name__ == "__main__":
savepoint_demo()
---
b.回滚性能优化
a.最小化回滚范围
通过合理的业务逻辑设计，减少需要回滚的操作数量，降低回滚操作的系统开销。
b.延迟验证策略
将复杂的验证操作放在事务后期执行，减少因早期验证失败导致的回滚开销。
c.批量操作回滚
对于批量数据处理，采用分段事务策略，避免单次回滚操作影响大量数据。
03.回滚监控与调试
a.回滚日志分析
监控和分析系统中的回滚操作，识别常见的回滚原因，优化业务逻辑和系统设计。
b.死锁检测与处理
实现死锁检测机制，当检测到死锁时自动选择合适的牺牲者进行回滚，确保系统可用性。
c.代码示例
---
# 回滚监控与死锁处理
import time
import threading
import sqlite3
from contextlib import contextmanager
class TransactionMonitor:
def __init__(self):
self.rollback_count = 0
self.deadlock_count = 0
self.lock = threading.Lock()
@contextmanager
def monitored_transaction(self, conn, name="transaction"):
"""带监控功能的事务上下文管理器"""
cursor = conn.cursor()
start_time = time.time()
try:
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print(f"[{name}] 事务开始")
yield cursor
conn.commit()
elapsed = time.time() - start_time
print(f"[{name}] ✅ 事务提交成功，耗时: {elapsed:.4f}秒")
except sqlite3.OperationalError as e:
if "database is locked" in str(e).lower():
with self.lock:
self.deadlock_count += 1
print(f"[{name}] ❌ 检测到死锁/锁冲突: {e}")
else:
print(f"[{name}] ❌ 操作错误: {e}")
conn.rollback()
with self.lock:
self.rollback_count += 1
print(f"[{name}] 事务已回滚")
raise
except Exception as e:
print(f"[{name}] ❌ 业务异常: {e}")
conn.rollback()
with self.lock:
self.rollback_count += 1
print(f"[{name}] 事务已回滚")
raise
def get_statistics(self):
"""获取监控统计信息"""
with self.lock:
return {
'total_rollbacks': self.rollback_count,
'deadlocks': self.deadlock_count
}
def reset_statistics(self):
"""重置统计信息"""
with self.lock:
self.rollback_count = 0
self.deadlock_count = 0
def deadlock_simulation_demo(self):
"""演示死锁检测和处理"""
monitor = self
def transaction_1():
conn = sqlite3.connect(':memory:', check_same_thread=False)
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE resources (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
owner TEXT
)
''')
cursor.executemany('INSERT INTO resources (name) VALUES (?)',
[('resource_A',), ('resource_B',)])
conn.commit()
try:
with monitor.monitored_transaction(conn, "事务1"):
# 获取资源A的锁
cursor.execute('UPDATE resources SET owner = ? WHERE name = ?', ('Tx1', 'resource_A'))
print("[事务1] 获取资源A")
time.sleep(1) # 模拟处理时间
# 尝试获取资源B的锁（可能导致死锁）
cursor.execute('UPDATE resources SET owner = ? WHERE name = ?', ('Tx1', 'resource_B'))
print("[事务1] 获取资源B")
# 模拟业务处理
time.sleep(0.5)
except Exception as e:
print(f"[事务1] 失败: {e}")
finally:
conn.close()
def transaction_2():
time.sleep(0.5) # 稍后启动
conn = sqlite3.connect(':memory:', check_same_thread=False)
cursor = conn.cursor()
# 创建相同的表结构
cursor.execute('''
CREATE TABLE resources (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
owner TEXT
)
''')
cursor.executemany('INSERT INTO resources (name) VALUES (?)',
[('resource_A',), ('resource_B',)])
conn.commit()
try:
with monitor.monitored_transaction(conn, "事务2"):
# 获取资源B的锁
cursor.execute('UPDATE resources SET owner = ? WHERE name = ?', ('Tx2', 'resource_B'))
print("[事务2] 获取资源B")
time.sleep(1) # 模拟处理时间
# 尝试获取资源A的锁（可能导致死锁）
cursor.execute('UPDATE resources SET owner = ? WHERE name = ?', ('Tx2', 'resource_A'))
print("[事务2] 获取资源A")
# 模拟业务处理
time.sleep(0.5)
except Exception as e:
print(f"[事务2] 失败: {e}")
finally:
conn.close()
# 启动两个并发事务
print("=== 死锁模拟演示 ===")
t1 = threading.Thread(target=transaction_1)
t2 = threading.Thread(target=transaction_2)
t1.start()
t2.start()
t1.join()
t2.join()
# 显示统计信息
stats = monitor.get_statistics()
print(f"\n监控统计:")
print(f" 总回滚次数: {stats['total_rollbacks']}")
print(f" 死锁次数: {stats['deadlocks']}")
# 使用示例
if __name__ == "__main__":
monitor = TransactionMonitor()
monitor.deadlock_simulation_demo()
---
1.4 自动提交模式
01.自动提交模式概述
a.模式定义
a.基本概念
自动提交模式是指每个SQL语句都被视为一个独立的事务，执行完毕后立即自动提交，无需显式调用COMMIT语句。
b.工作原理
在自动提交模式下，数据库驱动会在每个语句执行前自动开始事务，执行成功后立即提交，发生错误时自动回滚。
c.适用场景
适用于简单的单语句操作、批量数据处理、测试环境等场景，可以简化代码但牺牲了事务控制的灵活性。
b.模式切换机制
a.连接级设置
通过连接参数或特定的PRAGMA语句来控制整个连接的自动提交模式，影响该连接上的所有后续操作。
b.动态切换
在应用程序运行过程中，可以根据业务需要动态开启或关闭自动提交模式，提供更灵活的控制策略。
c.代码示例
---
# 自动提交模式演示
import sqlite3
import time
def autocommit_demo():
"""演示自动提交模式的特性和使用"""
print("=== 自动提交模式演示 ===")
# 创建测试数据库
conn = sqlite3.connect('autocommit_test.db')
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE test_records (
id INTEGER PRIMARY KEY,
data TEXT NOT NULL,
value REAL NOT NULL,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
conn.commit()
# 默认情况下，sqlite3的Python接口自动提交为False
print(f"当前连接的自动提交状态: {not conn.in_transaction}")
# 测试1：手动事务模式（默认）
print("\n--- 测试1：手动事务模式 ---")
cursor.execute('BEGIN TRANSACTION')
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('manual_1', 100))
print("插入第一条记录")
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('manual_2', 200))
print("插入第二条记录")
# 模拟错误
try:
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('manual_3', 'invalid'))
conn.commit()
print("事务提交成功")
except Exception as e:
print(f"发生错误: {e}")
conn.rollback()
print("事务已回滚")
# 检查结果
cursor.execute('SELECT COUNT(*) FROM test_records')
count = cursor.fetchone()[0]
print(f"测试后记录数: {count}") # 应该是0，因为事务被回滚了
# 测试2：自动提交模式
print("\n--- 测试2：启用自动提交模式 ---")
conn.autocommit = True # 注意：不是所有数据库驱动都支持这个属性
print("自动提交模式已启用")
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('auto_1', 300))
print("插入第一条记录（自动提交）")
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('auto_2', 400))
print("插入第二条记录（自动提交）")
# 模拟错误
try:
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('auto_3', 'invalid'))
print("插入第三条记录（自动提交）")
except Exception as e:
print(f"发生错误: {e}")
print("只有当前语句失败，前面插入的记录仍然存在")
# 检查结果
cursor.execute('SELECT COUNT(*) FROM test_records')
count = cursor.fetchone()[0]
print(f"测试后记录数: {count}") # 应该是2，因为每条语句都自动提交了
# 测试3：回到手动模式
print("\n--- 测试3：回到手动事务模式 ---")
conn.autocommit = False
cursor.execute('BEGIN TRANSACTION')
cursor.execute('INSERT INTO test_records (data, value) VALUES (?, ?)', ('manual_after_auto', 500))
conn.commit()
print("手动事务提交成功")
# 最终统计
cursor.execute('SELECT data, value FROM test_records ORDER BY id')
records = cursor.fetchall()
print("\n最终记录列表:")
for data, value in records:
print(f" {data}: {value}")
cursor.execute('SELECT COUNT(*) FROM test_records')
final_count = cursor.fetchone()[0]
print(f"最终记录总数: {final_count}")
conn.close()
# 清理测试文件
import os
if os.path.exists('autocommit_test.db'):
os.remove('autocommit_test.db')
if __name__ == "__main__":
autocommit_demo()
---
02.自动提交模式的优势与限制
a.主要优势
a.简化操作
无需显式管理事务的开始、提交和回滚，每个语句都是独立的操作单元，代码更加简洁。
b.减少锁定时间
每个语句执行完毕立即释放锁，减少资源占用时间，提高并发性能。
c.错误隔离
单个语句的失败不会影响其他语句，便于错误处理和恢复。
b.主要限制
a.缺乏原子性
多个相关操作无法作为一个整体进行管理，无法保证跨语句的数据一致性。
b.性能开销
每个语句都需要独立的提交过程，增加了系统开销，特别是在批量操作时。
c.无法回滚
一旦语句执行成功，就无法撤销，不适合需要复杂业务逻辑的场景。
c.代码示例
---
# 自动提交模式的性能和一致性对比
import time
import sqlite3
def autocommit_vs_manual_comparison():
"""对比自动提交和手动事务的性能和一致性"""
conn = sqlite3.connect('comparison_test.db')
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE performance_test (
id INTEGER PRIMARY KEY,
batch_id INTEGER,
data TEXT NOT NULL,
value REAL NOT NULL,
status TEXT DEFAULT 'active'
)
''')
conn.commit()
test_data = [
(f'item_{i}', i * 1.5) for i in range(1, 1001)
]
# 测试1：自动提交模式的批量插入
print("=== 测试1：自动提交模式批量插入 ===")
start_time = time.time()
conn.autocommit = True # 启用自动提交
successful_inserts = 0
failed_inserts = 0
for i, (data, value) in enumerate(test_data):
try:
cursor.execute('INSERT INTO performance_test (batch_id, data, value) VALUES (?, ?, ?)',
(1, data, value))
successful_inserts += 1
# 模拟10%的失败率
if i % 10 == 9:
cursor.execute('INSERT INTO performance_test (batch_id, data, value) VALUES (?, ?, ?)',
(1, 'invalid_data', 'not_a_number'))
except Exception as e:
failed_inserts += 1
print(f" 插入失败 (索引 {i}): {e}")
auto_commit_time = time.time() - start_time
cursor.execute('SELECT COUNT(*) FROM performance_test WHERE batch_id = 1')
auto_commit_count = cursor.fetchone()[0]
print(f"自动提交模式结果:")
print(f" 耗时: {auto_commit_time:.4f}秒")
print(f" 成功插入: {successful_inserts}条")
print(f" 失败插入: {failed_inserts}条")
print(f" 实际记录数: {auto_commit_count}条")
# 清空数据准备下次测试
cursor.execute('DELETE FROM performance_test')
conn.commit()
# 测试2：手动事务模式的批量插入
print("\n=== 测试2：手动事务模式批量插入 ===")
start_time = time.time()
conn.autocommit = False # 禁用自动提交
total_successful = 0
total_failed = 0
# 分批处理，每批50条记录
batch_size = 50
total_batches = len(test_data) // batch_size + (1 if len(test_data) % batch_size else 0)
for batch_num in range(total_batches):
start_idx = batch_num * batch_size
end_idx = min(start_idx + batch_size, len(test_data))
batch_data = test_data[start_idx:end_idx]
try:
cursor.execute('BEGIN TRANSACTION')
batch_successful = 0
for data, value in batch_data:
try:
cursor.execute('INSERT INTO performance_test (batch_id, data, value) VALUES (?, ?, ?)',
(2, data, value))
batch_successful += 1
# 模拟10%的失败率
if batch_successful % 10 == 0:
raise ValueError("模拟业务逻辑错误")
except Exception as e:
print(f" 批次{batch_num + 1}中记录失败: {e}")
conn.commit()
total_successful += batch_successful
print(f" 批次{batch_num + 1}: 成功{batch_successful}条")
except Exception as e:
conn.rollback()
total_failed += len(batch_data)
print(f" 批次{batch_num + 1}: 整批回滚 - {e}")
manual_commit_time = time.time() - start_time
cursor.execute('SELECT COUNT(*) FROM performance_test WHERE batch_id = 2')
manual_commit_count = cursor.fetchone()[0]
print(f"\n手动事务模式结果:")
print(f" 耗时: {manual_commit_time:.4f}秒")
print(f" 成功记录: {total_successful}条")
print(f" 失败记录: {total_failed}条")
print(f" 实际记录数: {manual_commit_count}条")
# 数据一致性验证
print(f"\n=== 数据一致性验证 ===")
cursor.execute('''
SELECT batch_id,
COUNT(*) as total_records,
COUNT(CASE WHEN value > 0 THEN 1 END) as valid_records,
AVG(value) as avg_value
FROM performance_test
GROUP BY batch_id
''')
results = cursor.fetchall()
for batch_id, total, valid, avg_val in results:
mode = "自动提交" if batch_id == 1 else "手动事务"
print(f"{mode} (批次{batch_id}): 总计{total}条，有效{valid}条，平均值{avg_val:.2f}")
conn.close()
# 清理测试文件
import os
if os.path.exists('comparison_test.db'):
os.remove('comparison_test.db')
if __name__ == "__main__":
autocommit_vs_manual_comparison()
---
03.自动提交模式的应用场景
a.开发与测试环境
a.快速原型开发
在开发阶段使用自动提交模式可以快速验证功能，无需关心事务管理的复杂性。
b.单元测试
单元测试中的简单数据操作可以使用自动提交模式，简化测试代码的编写和维护。
c.数据初始化
数据库初始化脚本通常使用自动提交模式，每条语句独立执行，便于定位问题。
b.生产环境应用
a.只读操作
对于查询、报表生成等只读操作，使用自动提交模式可以提高并发性能。
b.独立配置操作
系统配置、参数设置等独立的单语句操作适合使用自动提交模式。
c.批量数据处理
在确保数据一致性的前提下，某些批量数据导入操作可以使用自动提交模式。
c.代码示例
---
# 自动提交模式的实际应用场景
import sqlite3
import json
import logging
from datetime import datetime
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class DatabaseManager:
def __init__(self, db_path):
self.db_path = db_path
self.conn = None
self.setup_connection()
def setup_connection(self):
"""设置数据库连接"""
self.conn = sqlite3.connect(self.db_path)
self.conn.row_factory = sqlite3.Row # 使用Row工厂方便数据访问
self.conn.execute('PRAGMA foreign_keys = ON')
logger.info("数据库连接已建立")
def initialize_database(self):
"""数据库初始化 - 使用自动提交模式"""
logger.info("开始数据库初始化")
self.conn.autocommit = True # 启用自动提交模式
try:
# 创建用户表
self.conn.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT NOT NULL UNIQUE,
email TEXT NOT NULL UNIQUE,
created_at DATETIME DEFAULT CURRENT_TIMESTAMP,
is_active BOOLEAN DEFAULT 1
)
''')
# 创建配置表
self.conn.execute('''
CREATE TABLE IF NOT EXISTS app_config (
key TEXT PRIMARY KEY,
value TEXT NOT NULL,
description TEXT,
updated_at DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建操作日志表
self.conn.execute('''
CREATE TABLE IF NOT EXISTS operation_logs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
operation_type TEXT NOT NULL,
table_name TEXT NOT NULL,
record_id INTEGER,
old_value TEXT,
new_value TEXT,
user_id INTEGER,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
)
''')
# 插入默认配置
default_configs = [
('max_login_attempts', '5', '最大登录尝试次数'),
('session_timeout', '3600', '会话超时时间（秒）'),
('default_page_size', '20', '默认分页大小'),
('backup_retention_days', '30', '备份保留天数')
]
for key, value, desc in default_configs:
self.conn.execute('''
INSERT OR IGNORE INTO app_config (key, value, description)
VALUES (?, ?, ?)
''', (key, value, desc))
logger.info("数据库初始化完成")
except Exception as e:
logger.error(f"数据库初始化失败: {e}")
raise
finally:
self.conn.autocommit = False # 恢复手动模式
def log_operation(self, operation_type, table_name, record_id=None, old_value=None, new_value=None, user_id=None):
"""记录操作日志 - 使用自动提交模式确保日志及时写入"""
try:
original_autocommit = self.conn.autocommit
self.conn.autocommit = True
self.conn.execute('''
INSERT INTO operation_logs
(operation_type, table_name, record_id, old_value, new_value, user_id)
VALUES (?, ?, ?, ?, ?, ?)
''', (operation_type, table_name, record_id,
json.dumps(old_value) if old_value else None,
json.dumps(new_value) if new_value else None,
user_id))
except Exception as e:
logger.error(f"记录操作日志失败: {e}")
finally:
self.conn.autocommit = original_autocommit
def create_user(self, username, email):
"""创建用户 - 使用手动事务确保数据一致性"""
try:
self.conn.execute('BEGIN TRANSACTION')
# 检查用户是否已存在
cursor = self.conn.execute('SELECT id FROM users WHERE username = ? OR email = ?',
(username, email))
if cursor.fetchone():
raise ValueError("用户名或邮箱已存在")
# 插入新用户
cursor = self.conn.execute('''
INSERT INTO users (username, email)
VALUES (?, ?)
''', (username, email))
user_id = cursor.lastrowid
self.conn.commit()
logger.info(f"用户创建成功: {username} (ID: {user_id})")
# 记录日志（使用自动提交模式）
self.log_operation('CREATE', 'users', user_id, None,
{'username': username, 'email': email}, user_id)
return user_id
except Exception as e:
self.conn.rollback()
logger.error(f"创建用户失败: {e}")
self.log_operation('CREATE_FAILED', 'users', None, None,
{'username': username, 'email': email}, None)
raise
def update_config(self, key, value, user_id=None):
"""更新配置 - 使用自动提交模式"""
try:
self.conn.autocommit = True
# 获取旧值
cursor = self.conn.execute('SELECT value FROM app_config WHERE key = ?', (key,))
old_row = cursor.fetchone()
old_value = old_row['value'] if old_row else None
# 更新配置
self.conn.execute('''
UPDATE app_config
SET value = ?, updated_at = CURRENT_TIMESTAMP
WHERE key = ?
''', (value, key))
if self.conn.total_changes > 0:
logger.info(f"配置更新成功: {key} = {value}")
self.log_operation('UPDATE', 'app_config', key, old_value, value, user_id)
return True
else:
logger.warning(f"配置键不存在: {key}")
return False
except Exception as e:
logger.error(f"更新配置失败: {e}")
self.log_operation('UPDATE_FAILED', 'app_config', key, old_value, value, user_id)
raise
finally:
self.conn.autocommit = False
def generate_report(self, start_date=None, end_date=None):
"""生成操作报告 - 使用自动提交模式的只读操作"""
try:
self.conn.autocommit = True
query = '''
SELECT
operation_type,
table_name,
COUNT(*) as operation_count,
DATE(timestamp) as operation_date
FROM operation_logs
WHERE 1=1
'''
params = []
if start_date:
query += ' AND DATE(timestamp) >= ?'
params.append(start_date)
if end_date:
query += ' AND DATE(timestamp) <= ?'
params.append(end_date)
query += '''
GROUP BY operation_type, table_name, DATE(timestamp)
ORDER BY operation_date DESC, operation_type, table_name
'''
cursor = self.conn.execute(query, params)
report_data = cursor.fetchall()
logger.info(f"报告生成成功，共{len(report_data)}条记录")
return [dict(row) for row in report_data]
except Exception as e:
logger.error(f"生成报告失败: {e}")
raise
finally:
self.conn.autocommit = False
def close(self):
"""关闭数据库连接"""
if self.conn:
self.conn.close()
logger.info("数据库连接已关闭")
# 使用示例
def demo_autocommit_usage():
"""演示自动提交模式的实际应用"""
db_manager = DatabaseManager('demo_autocommit.db')
try:
# 初始化数据库（自动提交模式）
db_manager.initialize_database()
# 创建用户（手动事务模式）
user_id = db_manager.create_user('testuser', '[email protected]')
# 更新配置（自动提交模式）
db_manager.update_config('max_login_attempts', '10', user_id)
# 生成报告（自动提交模式的只读操作）
report = db_manager.generate_report()
print("操作报告:")
for item in report[:5]: # 只显示前5条
print(f" {item}")
except Exception as e:
print(f"演示失败: {e}")
finally:
db_manager.close()
# 清理演示文件
import os
if os.path.exists('demo_autocommit.db'):
os.remove('demo_autocommit.db')
if __name__ == "__main__":
demo_autocommit_usage()
---
2. SQLite事务
2.1 sqlite3模块
01.sqlite3模块概述
a.模块简介
a.内置数据库模块
sqlite3是Python标准库中的轻量级数据库模块，提供了完整的数据库功能，无需安装额外依赖，是Python内置的数据库解决方案。
b.嵌入式数据库特性
作为嵌入式数据库，sqlite3直接使用文件作为数据存储，无需独立的数据库服务器，适合轻量级应用和原型开发。
c.C语言接口绑定
sqlite3模块是对SQLite C库的Python绑定，保持了SQLite的高性能和可靠性，同时提供了Python风格的API接口。
b.核心特性
a.零配置需求
sqlite3无需复杂的配置过程，可以直接使用内存数据库或文件数据库，大大简化了开发部署流程。
b.事务支持
完整支持ACID事务特性，提供原子性、一致性、隔离性和持久性保证，适合需要数据完整性的应用场景。
c.跨平台兼容
数据库文件在不同操作系统之间完全兼容，支持Windows、Linux、macOS等主流平台。
c.代码示例
---
# sqlite3模块基础功能演示
import sqlite3
import os
import threading
from contextlib import contextmanager
from typing import Optional, List, Dict, Any
class SQLiteDemo:
def __init__(self, db_name: str = "demo.db"):
"""初始化SQLite演示类"""
self.db_name = db_name
self.connection_pool = {} # 简单的连接池
def basic_operations(self):
"""演示基本的数据库操作"""
print("=== SQLite基础操作演示 ===")
# 创建内存数据库连接
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建表
cursor.execute('''
CREATE TABLE employees (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
department TEXT NOT NULL,
salary REAL NOT NULL,
hire_date DATE NOT NULL
)
''')
# 插入数据
employees_data = [
('张三', '技术部', 8000.0, '2023-01-15'),
('李四', '销售部', 6000.0, '2023-02-20'),
('王五', '技术部', 9000.0, '2023-03-10'),
('赵六', '人事部', 5500.0, '2023-04-05')
]
cursor.executemany('''
INSERT INTO employees (name, department, salary, hire_date)
VALUES (?, ?, ?, ?)
''', employees_data)
conn.commit()
print(f"插入了 {len(employees_data)} 条员工记录")
# 查询数据
cursor.execute('SELECT * FROM employees')
employees = cursor.fetchall()
print("\n员工列表:")
for emp in employees:
print(f" ID:{emp[0]}, 姓名:{emp[1]}, 部门:{emp[2]}, 薪资:{emp[3]}, 入职日期:{emp[4]}")
# 聚合查询
cursor.execute('''
SELECT department, COUNT(*) as count, AVG(salary) as avg_salary
FROM employees
GROUP BY department
''')
dept_stats = cursor.fetchall()
print("\n部门统计:")
for dept, count, avg_salary in dept_stats:
print(f" {dept}: {count}人, 平均薪资: {avg_salary:.2f}")
conn.close()
def transaction_demo(self):
"""演示事务操作"""
print("\n=== 事务操作演示 ===")
conn = sqlite3.connect(':memory:')
cursor = conn.cursor()
# 创建银行账户表
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
account_number TEXT UNIQUE NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
# 插入测试账户
cursor.executemany('''
INSERT INTO accounts (account_number, balance)
VALUES (?, ?)
''', [('A001', 1000.0), ('A002', 2000.0), ('A003', 1500.0)])
conn.commit()
# 开始转账事务
try:
cursor.execute('BEGIN TRANSACTION')
# 从A001转账200到A002
cursor.execute('''
UPDATE accounts SET balance = balance - 200
WHERE account_number = ?
''', ('A001',))
cursor.execute('''
UPDATE accounts SET balance = balance + 200
WHERE account_number = ?
''', ('A002',))
# 验证转账结果
cursor.execute('SELECT account_number, balance FROM accounts WHERE account_number IN (?, ?)',
('A001', 'A002'))
accounts = cursor.fetchall()
print("转账后账户状态:")
for acc_num, balance in accounts:
print(f" {acc_num}: {balance:.2f}")
conn.commit()
print("✅ 转账成功！")
except Exception as e:
conn.rollback()
print(f"❌ 转账失败: {e}")
finally:
conn.close()
if __name__ == "__main__":
demo = SQLiteDemo()
demo.basic_operations()
demo.transaction_demo()
---
02.连接管理
a.数据库连接类型
a.内存数据库连接
使用':memory:'作为连接字符串创建内存数据库，数据仅存在于内存中，连接关闭后数据消失，适合临时数据处理和测试场景。
b.文件数据库连接
使用文件路径作为连接字符串创建持久化数据库，数据保存在文件中，适合需要长期存储的应用场景。
c.特殊连接模式
支持只读模式、临时文件模式等特殊连接方式，满足不同的应用需求和安全性要求。
b.连接参数配置
a.超时设置
通过timeout参数设置SQL执行超时时间，防止长时间运行的查询阻塞应用程序，提高系统响应性。
b.检查相同线程
check_same_thread参数控制连接是否只能被创建线程使用，影响多线程环境下的连接管理策略。
c.隔离级别设置
通过isolation_level参数控制事务的自动提交行为和隔离级别，适应不同的并发访问需求。
c.代码示例
---
# 数据库连接管理演示
import sqlite3
import time
import threading
from typing import Optional, Dict, Any
import tempfile
import os
class ConnectionManager:
def __init__(self):
"""连接管理器初始化"""
self.connections: Dict[str, sqlite3.Connection] = {}
self.lock = threading.Lock()
def create_memory_database(self) -> sqlite3.Connection:
"""创建内存数据库连接"""
print("创建内存数据库连接...")
conn = sqlite3.connect(':memory:')
# 配置连接参数
conn.execute('PRAGMA foreign_keys = ON') # 启用外键约束
conn.execute('PRAGMA journal_mode = MEMORY') # 内存日志模式
# 创建示例数据
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE test_data (
id INTEGER PRIMARY KEY,
message TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
INSERT INTO test_data (message) VALUES (?)
''', ('内存数据库创建成功',))
conn.commit()
print("✅ 内存数据库创建完成")
return conn
def create_file_database(self, db_path: str, read_only: bool = False) -> sqlite3.Connection:
"""创建文件数据库连接"""
print(f"创建文件数据库连接: {db_path}")
if read_only:
# 只读模式连接
uri = f"file:{db_path}?mode=ro"
conn = sqlite3.connect(uri, uri=True)
print("🔒 以只读模式连接")
else:
# 读写模式连接
conn = sqlite3.connect(db_path)
# 优化配置
conn.execute('PRAGMA foreign_keys = ON')
conn.execute('PRAGMA journal_mode = WAL') # WAL模式提高并发性能
conn.execute('PRAGMA synchronous = NORMAL') # 平衡性能和安全性
conn.execute('PRAGMA cache_size = 10000') # 增加缓存大小
# 初始化数据库结构
self._initialize_database(conn)
print("✅ 文件数据库创建完成")
return conn
def _initialize_database(self, conn: sqlite3.Connection):
"""初始化数据库结构"""
cursor = conn.cursor()
# 创建用户表
cursor.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT UNIQUE NOT NULL,
email TEXT UNIQUE NOT NULL,
password_hash TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_login TIMESTAMP,
is_active BOOLEAN DEFAULT 1
)
''')
# 创建产品表
cursor.execute('''
CREATE TABLE IF NOT EXISTS products (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
description TEXT,
price REAL NOT NULL CHECK(price >= 0),
stock_quantity INTEGER NOT NULL DEFAULT 0 CHECK(stock_quantity >= 0),
category_id INTEGER,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建订单表
cursor.execute('''
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
order_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
total_amount REAL NOT NULL CHECK(total_amount >= 0),
status TEXT NOT NULL DEFAULT 'pending',
shipping_address TEXT,
FOREIGN KEY (user_id) REFERENCES users(id)
)
''')
# 创建订单项表
cursor.execute('''
CREATE TABLE IF NOT EXISTS order_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
order_id INTEGER NOT NULL,
product_id INTEGER NOT NULL,
quantity INTEGER NOT NULL CHECK(quantity > 0),
unit_price REAL NOT NULL CHECK(unit_price >= 0),
total_price REAL NOT NULL CHECK(total_price >= 0),
FOREIGN KEY (order_id) REFERENCES orders(id),
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
conn.commit()
print("📋 数据库结构初始化完成")
def create_temp_database(self) -> sqlite3.Connection:
"""创建临时数据库连接"""
# 创建临时文件
temp_fd, temp_path = tempfile.mkstemp(suffix='.db')
os.close(temp_fd) # 关闭文件描述符
print(f"创建临时数据库: {temp_path}")
conn = sqlite3.connect(temp_path)
# 注册删除临时文件的函数
def cleanup_temp_db():
try:
conn.close()
os.unlink(temp_path)
print(f"🗑️ 临时数据库已删除: {temp_path}")
except Exception as e:
print(f"删除临时数据库失败: {e}")
# 将清理函数绑定到连接对象
conn._cleanup = cleanup_temp_db
return conn
def connection_with_parameters(self):
"""演示不同连接参数的使用"""
print("\n=== 连接参数配置演示 ===")
# 参数1：设置超时时间
print("1. 设置查询超时时间")
try:
conn = sqlite3.connect(':memory:', timeout=5.0) # 5秒超时
cursor = conn.cursor()
cursor.execute('CREATE TABLE timeout_test (data TEXT)')
start_time = time.time()
# 模拟可能超时的操作（SQLite通常不会在这里超时，只是演示）
cursor.execute("SELECT 1")
end_time = time.time()
print(f" 查询执行时间: {end_time - start_time:.4f}秒")
conn.close()
except sqlite3.OperationalError as e:
print(f" ❌ 查询超时: {e}")
# 参数2：线程安全设置
print("\n2. 线程安全设置测试")
def thread_worker(thread_id: int):
try:
# check_same_thread=False允许跨线程使用
conn = sqlite3.connect(':memory:', check_same_thread=False)
cursor = conn.cursor()
cursor.execute('CREATE TABLE thread_test (thread_id INTEGER, data TEXT)')
cursor.execute('INSERT INTO thread_test VALUES (?, ?)',
(thread_id, f"Thread {thread_id} data"))
cursor.execute('SELECT * FROM thread_test')
result = cursor.fetchone()
print(f" 线程 {thread_id}: {result}")
conn.close()
except Exception as e:
print(f" 线程 {thread_id} 错误: {e}")
# 创建多个线程测试
threads = []
for i in range(3):
thread = threading.Thread(target=thread_worker, args=(i,))
threads.append(thread)
thread.start()
for thread in threads:
thread.join()
# 参数3：自动提交设置
print("\n3. 自动提交设置测试")
# 隐式事务模式（默认）
conn1 = sqlite3.connect(':memory:')
cursor1 = conn1.cursor()
cursor1.execute('CREATE TABLE auto_commit_test (id INTEGER, value TEXT)')
cursor1.execute('INSERT INTO auto_commit_test VALUES (1, "test")')
conn1.commit() # 需要手动提交
print(" 隐式事务模式：需要手动提交")
# 自动提交模式
conn2 = sqlite3.connect(':memory:', isolation_level=None)
cursor2 = conn2.cursor()
cursor2.execute('CREATE TABLE auto_commit_test (id INTEGER, value TEXT)')
cursor2.execute('INSERT INTO auto_commit_test VALUES (1, "auto")')
print(" 自动提交模式：立即生效")
conn1.close()
conn2.close()
def connection_pool_demo(self):
"""演示简单的连接池实现"""
print("\n=== 连接池演示 ===")
class SimpleConnectionPool:
def __init__(self, max_connections: int = 5):
self.max_connections = max_connections
self.pool = []
self.lock = threading.Lock()
self.created_connections = 0
def get_connection(self, db_path: str) -> sqlite3.Connection:
with self.lock:
# 检查池中是否有可用连接
if self.pool:
conn = self.pool.pop()
print(f" 📤 从池中获取连接 (池中剩余: {len(self.pool)})")
return conn
# 如果池为空且未达到最大连接数，创建新连接
if self.created_connections < self.max_connections:
conn = sqlite3.connect(db_path)
self.created_connections += 1
print(f" 🔨 创建新连接 (已创建: {self.created_connections})")
return conn
# 如果达到最大连接数，等待或抛出异常
raise Exception("连接池已满")
def return_connection(self, conn: sqlite3.Connection):
with self.lock:
self.pool.append(conn)
print(f" 📥 连接返回池中 (池中当前: {len(self.pool)})")
def close_all(self):
with self.lock:
while self.pool:
conn = self.pool.pop()
conn.close()
print(" 🔒 连接池已关闭")
# 创建连接池
pool = SimpleConnectionPool(max_connections=3)
# 模拟多线程使用连接池
def pool_worker(worker_id: int):
try:
print(f"工作线程 {worker_id} 请求连接")
conn = pool.get_connection(':memory:')
# 使用连接执行一些操作
cursor = conn.cursor()
cursor.execute('CREATE TABLE pool_test (worker_id INTEGER, data TEXT)')
cursor.execute('INSERT INTO pool_test VALUES (?, ?)',
(worker_id, f"Worker {worker_id} data"))
# 模拟工作时间
time.sleep(0.1)
result = cursor.execute('SELECT * FROM pool_test').fetchone()
print(f" 工作线程 {worker_id}: {result}")
# 返回连接到池
pool.return_connection(conn)
print(f"工作线程 {worker_id} 释放连接")
except Exception as e:
print(f"工作线程 {worker_id} 错误: {e}")
# 启动多个工作线程
workers = []
for i in range(5):
worker = threading.Thread(target=pool_worker, args=(i,))
workers.append(worker)
worker.start()
for worker in workers:
worker.join()
pool.close_all()
if __name__ == "__main__":
manager = ConnectionManager()
# 演示不同类型的数据库连接
mem_conn = manager.create_memory_database()
# 查询内存数据库数据
cursor = mem_conn.cursor()
cursor.execute('SELECT * FROM test_data')
result = cursor.fetchone()
print(f"内存数据库数据: {result}")
mem_conn.close()
# 演示文件数据库连接
file_conn = manager.create_file_database("test_database.db")
file_conn.close()
# 演示临时数据库
temp_conn = manager.create_temp_database()
temp_conn._cleanup() # 手动清理
# 演示连接参数
manager.connection_with_parameters()
# 演示连接池
manager.connection_pool_demo()
# 清理测试文件
if os.path.exists("test_database.db"):
os.remove("test_database.db")
print("🗑️ 测试数据库文件已删除")
---
03.高级特性与接口
a.事务控制接口
a.显式事务控制
提供BEGIN、COMMIT、ROLLBACK等SQL语句的直接执行接口，允许精确控制事务的开始、提交和回滚过程。
b.上下文管理器
通过contextlib提供的上下文管理器模式，自动处理事务的开始和结束，简化代码并确保资源正确释放。
c.保存点机制
支持在事务中设置多个保存点，允许部分回滚操作，提高复杂业务逻辑的灵活性和性能。
b.性能优化特性
a.预编译语句
支持SQL语句的预编译和缓存，减少SQL解析开销，提高重复执行相同SQL语句的性能。
b.批量操作接口
提供executemany等批量操作接口，优化大量数据的插入、更新、删除操作，显著提升数据处理效率。
c.WAL模式支持
支持Write-Ahead Logging模式，提高并发读写性能，允许读操作与写操作同时进行。
c.代码示例
---
# 高级特性与接口演示
import sqlite3
import time
from contextlib import contextmanager
from typing import List, Tuple, Optional, Any
import threading
class SQLiteAdvancedFeatures:
def __init__(self, db_path: str = ":memory:"):
"""初始化高级特性演示类"""
self.db_path = db_path
self.connection = sqlite3.connect(db_path)
self._setup_database()
def _setup_database(self):
"""设置数据库结构"""
cursor = self.connection.cursor()
# 创建综合测试表
cursor.execute('''
CREATE TABLE performance_test (
id INTEGER PRIMARY KEY,
category TEXT NOT NULL,
name TEXT NOT NULL,
value REAL NOT NULL,
description TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建索引
cursor.execute('CREATE INDEX idx_category ON performance_test(category)')
cursor.execute('CREATE INDEX idx_name ON performance_test(name)')
self.connection.commit()
print("📋 数据库结构设置完成")
@contextmanager
def transaction_context(self, isolation_level: str = "DEFERRED"):
"""事务上下文管理器"""
cursor = self.connection.cursor()
try:
# 开始事务
if isolation_level == "IMMEDIATE":
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
elif isolation_level == "EXCLUSIVE":
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
else:
cursor.execute('BEGIN DEFERRED TRANSACTION')
print(f"🚀 事务开始 (隔离级别: {isolation_level})")
yield cursor
# 提交事务
self.connection.commit()
print("✅ 事务提交成功")
except Exception as e:
# 回滚事务
self.connection.rollback()
print(f"❌ 事务回滚: {e}")
raise
def savepoint_demo(self):
"""演示保存点机制"""
print("\n=== 保存点机制演示 ===")
with self.transaction_context() as cursor:
# 创建测试表
cursor.execute('''
CREATE TABLE savepoint_test (
id INTEGER PRIMARY KEY,
step INTEGER NOT NULL,
data TEXT NOT NULL
)
''')
# 插入初始数据
cursor.execute('INSERT INTO savepoint_test (step, data) VALUES (1, "初始数据")')
print("📝 插入初始数据")
# 设置第一个保存点
cursor.execute('SAVEPOINT step1_complete')
print("📍 设置保存点: step1_complete")
# 插入第二批数据
cursor.executemany('INSERT INTO savepoint_test (step, data) VALUES (?, ?)',
[(2, "第二批数据1"), (2, "第二批数据2")])
print("📝 插入第二批数据")
# 设置第二个保存点
cursor.execute('SAVEPOINT step2_complete')
print("📍 设置保存点: step2_complete")
# 尝试插入可能失败的数据
try:
# 模拟违反约束的操作
cursor.execute('INSERT INTO savepoint_test (step, data) VALUES (?, ?)',
(999, "特殊数据"))
print("📝 插入特殊数据")
# 如果一切正常，设置第三个保存点
cursor.execute('SAVEPOINT step3_complete')
print("📍 设置保存点: step3_complete")
except Exception as e:
print(f"❌ 插入特殊数据失败: {e}")
print("↩️ 回滚到保存点: step2_complete")
cursor.execute('ROLLBACK TO SAVEPOINT step2_complete')
# 继续正常操作
cursor.execute('INSERT INTO savepoint_test (step, data) VALUES (?, ?)',
(3, "第三批数据"))
print("📝 插入第三批数据")
# 查询最终结果
cursor.execute('SELECT step, COUNT(*) as count FROM savepoint_test GROUP BY step ORDER BY step')
results = cursor.fetchall()
print("\n📊 最终数据统计:")
for step, count in results:
print(f" 步骤 {step}: {count} 条记录")
# 释放所有保存点
cursor.execute('RELEASE SAVEPOINT step1_complete')
cursor.execute('RELEASE SAVEPOINT step2_complete')
print("🔓 释放所有保存点")
def prepared_statement_demo(self):
"""演示预编译语句"""
print("\n=== 预编译语句演示 ===")
# 准备测试数据
test_data = [
("电子产品", "智能手机", 2999.99, "最新款智能手机"),
("电子产品", "笔记本电脑", 6999.99, "高性能笔记本"),
("服装", "T恤", 99.99, "纯棉T恤"),
("服装", "牛仔裤", 299.99, "修身牛仔裤"),
("图书", "Python编程", 89.99, "Python入门书籍"),
("图书", "数据结构", 79.99, "算法与数据结构")
]
# 方法1：使用参数化查询
print("1. 使用参数化查询")
start_time = time.time()
insert_sql = 'INSERT INTO performance_test (category, name, value, description) VALUES (?, ?, ?, ?)'
for i, (category, name, value, description) in enumerate(test_data * 100): # 重复100次
self.connection.execute(insert_sql, (category, name, value, description))
self.connection.commit()
param_time = time.time() - start_time
print(f" 参数化查询耗时: {param_time:.4f}秒")
# 清空数据
cursor = self.connection.cursor()
cursor.execute('DELETE FROM performance_test')
self.connection.commit()
# 方法2：使用预编译语句（通过cursor.prepare模拟）
print("2. 使用预编译语句模拟")
start_time = time.time()
# 创建语句对象并重用
statement = self.connection.cursor()
for i, (category, name, value, description) in enumerate(test_data * 100):
statement.execute(insert_sql, (category, name, value, description))
self.connection.commit()
prepared_time = time.time() - start_time
print(f" 预编译语句耗时: {prepared_time:.4f}秒")
# 性能对比
improvement = (param_time - prepared_time) / param_time * 100
print(f" 📈 性能提升: {improvement:.2f}%")
# 查询结果
cursor.execute('SELECT category, COUNT(*) FROM performance_test GROUP BY category')
results = cursor.fetchall()
print("\n📊 数据统计:")
for category, count in results:
print(f" {category}: {count} 条记录")
def batch_operations_demo(self):
"""演示批量操作"""
print("\n=== 批量操作演示 ===")
cursor = self.connection.cursor()
# 清空之前的数据
cursor.execute('DELETE FROM performance_test')
self.connection.commit()
# 准备大量测试数据
batch_size = 10000
print(f"准备生成 {batch_size} 条测试数据...")
test_data = []
categories = ["电子产品", "服装", "图书", "家居", "食品"]
for i in range(batch_size):
category = categories[i % len(categories)]
test_data.append((
category,
f"产品_{i}",
float(i * 0.1),
f"这是第{i}个产品的描述"
))
# 方法1：逐条插入
print("1. 逐条插入测试")
cursor.execute('DELETE FROM performance_test')
self.connection.commit()
start_time = time.time()
for data in test_data[:1000]: # 只测试1000条
cursor.execute('INSERT INTO performance_test (category, name, value, description) VALUES (?, ?, ?, ?)', data)
self.connection.commit()
single_time = time.time() - start_time
print(f" 1000条记录逐条插入耗时: {single_time:.4f}秒")
# 方法2：批量插入
print("2. 批量插入测试")
cursor.execute('DELETE FROM performance_test')
self.connection.commit()
start_time = time.time()
cursor.executemany('INSERT INTO performance_test (category, name, value, description) VALUES (?, ?, ?, ?)', test_data)
self.connection.commit()
batch_time = time.time() - start_time
print(f" {batch_size}条记录批量插入耗时: {batch_time:.4f}秒")
# 性能对比
if single_time > 0:
estimated_single_total = single_time * 10 # 估算10000条的时间
improvement = (estimated_single_total - batch_time) / estimated_single_total * 100
print(f" 📈 批量操作性能提升: {improvement:.2f}%")
# 方法3：分批次插入
print("3. 分批次插入测试")
cursor.execute('DELETE FROM performance_test')
self.connection.commit()
chunk_size = 1000
start_time = time.time()
with self.transaction_context():
for i in range(0, len(test_data), chunk_size):
chunk = test_data[i:i + chunk_size]
cursor.executemany('INSERT INTO performance_test (category, name, value, description) VALUES (?, ?, ?, ?)', chunk)
if (i + chunk_size) % 5000 == 0:
print(f" 已插入 {i + chunk_size} 条记录...")
self.connection.commit()
cursor.execute('BEGIN TRANSACTION')
chunk_time = time.time() - start_time
print(f" {batch_size}条记录分批次插入耗时: {chunk_time:.4f}秒")
# 验证数据
cursor.execute('SELECT COUNT(*) FROM performance_test')
total_count = cursor.fetchone()[0]
print(f" ✅ 总记录数: {total_count}")
# 聚合查询测试
cursor.execute('''
SELECT category, COUNT(*) as count, AVG(value) as avg_value, MIN(value) as min_value, MAX(value) as max_value
FROM performance_test
GROUP BY category
ORDER BY count DESC
''')
stats = cursor.fetchall()
print("\n📊 分类统计:")
for category, count, avg_val, min_val, max_val in stats:
print(f" {category}: {count}条, 平均值:{avg_val:.2f}, 范围:{min_val:.1f}-{max_val:.1f}")
def wal_mode_demo(self):
"""演示WAL模式"""
print("\n=== WAL模式演示 ===")
# 创建新的数据库连接用于WAL测试
wal_conn = sqlite3.connect('wal_test.db')
cursor = wal_conn.cursor()
# 设置WAL模式
cursor.execute('PRAGMA journal_mode = WAL')
cursor.execute('PRAGMA synchronous = NORMAL')
cursor.execute('PRAGMA cache_size = 10000')
# 创建测试表
cursor.execute('''
CREATE TABLE wal_test (
id INTEGER PRIMARY KEY,
data TEXT NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
wal_conn.commit()
print("✅ WAL模式数据库创建完成")
# 并发读写测试
def writer_thread(thread_id: int, write_count: int):
"""写线程"""
try:
conn = sqlite3.connect('wal_test.db')
cursor = conn.cursor()
for i in range(write_count):
cursor.execute('INSERT INTO wal_test (data) VALUES (?)',
(f'Writer {thread_id} - Message {i}',))
if i % 10 == 0:
conn.commit()
print(f" 写线程 {thread_id}: 已写入 {i+1} 条记录")
conn.commit()
print(f" ✅ 写线程 {thread_id} 完成")
conn.close()
except Exception as e:
print(f" ❌ 写线程 {thread_id} 错误: {e}")
def reader_thread(thread_id: int, read_duration: float):
"""读线程"""
try:
conn = sqlite3.connect('wal_test.db')
cursor = conn.cursor()
start_time = time.time()
read_count = 0
while time.time() - start_time < read_duration:
cursor.execute('SELECT COUNT(*) FROM wal_test')
count = cursor.fetchone()[0]
read_count += 1
time.sleep(0.01) # 避免过度占用CPU
print(f" ✅ 读线程 {thread_id}: 完成 {read_count} 次读取")
conn.close()
except Exception as e:
print(f" ❌ 读线程 {thread_id} 错误: {e}")
# 启动并发测试
print("🚀 启动并发读写测试...")
threads = []
# 启动2个写线程
for i in range(2):
writer = threading.Thread(target=writer_thread, args=(i+1, 50))
threads.append(writer)
writer.start()
# 启动3个读线程
for i in range(3):
reader = threading.Thread(target=reader_thread, args=(i+1, 2.0))
threads.append(reader)
reader.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 查询最终结果
cursor.execute('SELECT COUNT(*) FROM wal_test')
final_count = cursor.fetchone()[0]
print(f"\n📊 最终记录数: {final_count}")
# 清理
wal_conn.close()
import os
if os.path.exists('wal_test.db'):
os.remove('wal_test.db')
if os.path.exists('wal_test.db-wal'):
os.remove('wal_test.db-wal')
if os.path.exists('wal_test.db-shm'):
os.remove('wal_test.db-shm')
print("🗑️ WAL测试文件已清理")
if __name__ == "__main__":
# 创建演示实例
demo = SQLiteAdvancedFeatures("advanced_features_demo.db")
# 演示事务上下文管理器
print("=== 事务上下文管理器演示 ===")
with demo.transaction_context("IMMEDIATE") as cursor:
cursor.execute('''
INSERT INTO performance_test (category, name, value, description)
VALUES (?, ?, ?, ?)
''', ("测试", "上下文管理器测试", 100.0, "使用上下文管理器插入的数据"))
cursor.execute('SELECT * FROM performance_test')
result = cursor.fetchone()
print(f" 插入数据: {result}")
# 演示保存点
demo.savepoint_demo()
# 演示预编译语句
demo.prepared_statement_demo()
# 演示批量操作
demo.batch_operations_demo()
# 演示WAL模式
demo.wal_mode_demo()
# 清理演示数据库
demo.connection.close()
import os
if os.path.exists("advanced_features_demo.db"):
os.remove("advanced_features_demo.db")
print("🗑️ 演示数据库文件已清理")
---
04.最佳实践与安全
a.安全编程实践
a.SQL注入防护
始终使用参数化查询而不是字符串拼接来构建SQL语句，有效防止SQL注入攻击，保护数据库安全。
b.连接安全配置
合理配置数据库连接参数，包括设置适当的超时时间、限制连接访问权限、启用外键约束等安全措施。
c.数据验证与清理
在数据写入数据库前进行严格的验证和清理，确保数据的完整性和安全性，防止恶意数据破坏系统。
b.性能优化建议
a.索引策略
根据查询模式合理创建索引，平衡查询性能和写入性能，避免过度索引导致的性能下降。
b.查询优化
使用EXPLAIN QUERY PLAN分析查询执行计划，优化SQL语句结构，避免全表扫描和不必要的计算。
c.内存管理
合理设置缓存大小和连接池参数，避免内存泄漏和过度消耗，提高系统整体性能。
c.代码示例
---
# 最佳实践与安全演示
import sqlite3
import hashlib
import secrets
import time
import re
from contextlib import contextmanager
from typing import Optional, List, Dict, Any, Tuple
import logging
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class SQLiteBestPractices:
def __init__(self, db_path: str = "secure_app.db"):
"""初始化安全实践演示类"""
self.db_path = db_path
self.connection = None
self._initialize_secure_database()
def _initialize_secure_database(self):
"""初始化安全数据库配置"""
try:
# 创建安全配置的数据库连接
self.connection = sqlite3.connect(
self.db_path,
timeout=30.0, # 30秒超时
check_same_thread=False, # 线程安全
isolation_level="DEFERRED" # 默认隔离级别
)
# 安全配置
cursor = self.connection.cursor()
cursor.execute('PRAGMA foreign_keys = ON') # 启用外键约束
cursor.execute('PRAGMA journal_mode = WAL') # WAL模式
cursor.execute('PRAGMA synchronous = NORMAL') # 平衡性能和安全
cursor.execute('PRAGMA secure_delete = ON') # 安全删除
cursor.execute('PRAGMA temp_store = MEMORY') # 临时表存储在内存
cursor.execute('PRAGMA mmap_size = 268435456') # 256MB内存映射
# 创建安全的表结构
self._create_secure_tables()
self.connection.commit()
logger.info("✅ 安全数据库初始化完成")
except Exception as e:
logger.error(f"❌ 数据库初始化失败: {e}")
raise
def _create_secure_tables(self):
"""创建安全的表结构"""
cursor = self.connection.cursor()
# 用户表（带安全约束）
cursor.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT UNIQUE NOT NULL CHECK(length(username) >= 3 AND length(username) <= 50),
email TEXT UNIQUE NOT NULL CHECK(email LIKE '%@%.%'),
password_hash TEXT NOT NULL CHECK(length(password_hash) >= 64),
salt TEXT NOT NULL CHECK(length(salt) == 32),
is_active BOOLEAN DEFAULT 1 CHECK(is_active IN (0, 1)),
failed_login_attempts INTEGER DEFAULT 0 CHECK(failed_login_attempts >= 0),
account_locked BOOLEAN DEFAULT 0 CHECK(account_locked IN (0, 1)),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
last_login TIMESTAMP,
login_count INTEGER DEFAULT 0 CHECK(login_count >= 0)
)
''')
# 审计日志表
cursor.execute('''
CREATE TABLE IF NOT EXISTS audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER,
action TEXT NOT NULL CHECK(length(action) > 0),
table_name TEXT,
record_id INTEGER,
old_values TEXT,
new_values TEXT,
ip_address TEXT,
user_agent TEXT,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
)
''')
# 敏感数据表（加密字段）
cursor.execute('''
CREATE TABLE IF NOT EXISTS sensitive_data (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
data_type TEXT NOT NULL,
encrypted_content TEXT NOT NULL,
content_hash TEXT NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
expires_at TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
)
''')
# 创建安全索引
cursor.execute('CREATE INDEX IF NOT EXISTS idx_users_username ON users(username)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_users_email ON users(email)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_audit_user_action ON audit_log(user_id, action)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_audit_timestamp ON audit_log(timestamp)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_sensitive_user_type ON sensitive_data(user_id, data_type)')
logger.info("📋 安全表结构创建完成")
def secure_password_handling(self):
"""演示安全密码处理"""
print("\n=== 安全密码处理演示 ===")
def generate_salt() -> str:
"""生成安全的盐值"""
return secrets.token_hex(16) # 32字符的十六进制盐值
def hash_password(password: str, salt: str) -> str:
"""安全的密码哈希"""
# 使用PBKDF2算法进行密码哈希
password_bytes = password.encode('utf-8')
salt_bytes = salt.encode('utf-8')
# 进行100,000次迭代
hash_obj = hashlib.pbkdf2_hmac('sha256', password_bytes, salt_bytes, 100000)
return hash_obj.hex()
def verify_password(password: str, salt: str, stored_hash: str) -> bool:
"""验证密码"""
computed_hash = hash_password(password, salt)
return secrets.compare_digest(computed_hash, stored_hash)
# 测试用户数据
test_users = [
("alice", "[email protected]", "SecurePass123!"),
("bob", "[email protected]", "StrongPass456@"),
("charlie", "[email protected]", "SafePass789#")
]
cursor = self.connection.cursor()
for username, email, password in test_users:
try:
# 生成盐值和哈希
salt = generate_salt()
password_hash = hash_password(password, salt)
# 安全插入用户数据（使用参数化查询）
cursor.execute('''
INSERT INTO users (username, email, password_hash, salt)
VALUES (?, ?, ?, ?)
''', (username, email, password_hash, salt))
print(f"✅ 用户 {username} 创建成功")
# 验证密码
cursor.execute('SELECT password_hash, salt FROM users WHERE username = ?', (username,))
stored_hash, stored_salt = cursor.fetchone()
# 正确密码验证
is_valid = verify_password(password, stored_salt, stored_hash)
print(f" 密码验证 (正确): {is_valid}")
# 错误密码验证
is_invalid = verify_password("wrongpassword", stored_salt, stored_hash)
print(f" 密码验证 (错误): {is_invalid}")
except Exception as e:
print(f"❌ 创建用户 {username} 失败: {e}")
self.connection.commit()
print("🔐 密码处理演示完成")
def sql_injection_prevention(self):
"""演示SQL注入防护"""
print("\n=== SQL注入防护演示 ===")
cursor = self.connection.cursor()
# 创建演示表
cursor.execute('''
CREATE TABLE IF NOT EXISTS products (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
price REAL NOT NULL CHECK(price >= 0),
category TEXT NOT NULL
)
''')
# 插入测试数据
products = [
("智能手机", 2999.99, "电子产品"),
("笔记本电脑", 6999.99, "电子产品"),
("T恤", 99.99, "服装")
]
cursor.executemany('INSERT INTO products (name, price, category) VALUES (?, ?, ?)', products)
self.connection.commit()
# 安全的查询方法
def safe_search_products(category: str, min_price: float = 0) -> List[Tuple]:
"""安全的产品搜索（防SQL注入）"""
try:
# 使用参数化查询
cursor.execute('''
SELECT id, name, price, category
FROM products
WHERE category = ? AND price >= ?
ORDER BY price DESC
''', (category, min_price))
return cursor.fetchall()
except Exception as e:
logger.error(f"查询失败: {e}")
return []
# 危险的查询方法（仅用于演示）
def dangerous_search_products(category: str) -> List[Tuple]:
"""危险的产品搜索（易受SQL注入攻击）"""
try:
# 危险：直接字符串拼接
query = f"SELECT id, name, price, category FROM products WHERE category = '{category}'"
print(f"⚠️ 危险查询: {query}")
cursor.execute(query)
return cursor.fetchall()
except Exception as e:
logger.error(f"查询失败: {e}")
return []
# 测试正常查询
print("1. 正常查询测试:")
safe_results = safe_search_products("电子产品")
print(" 安全查询结果:")
for product in safe_results:
print(f" {product[1]} - ¥{product[2]}")
# 测试SQL注入攻击
print("\n2. SQL注入攻击测试:")
malicious_input = "电子产品'; DROP TABLE products; --"
print(f"恶意输入: {malicious_input}")
# 安全查询的防护
print("安全查询防护:")
safe_results = safe_search_products(malicious_input)
print(f" 结果: {safe_results} (空结果，但表安全)")
# 危险查询的漏洞
print("危险查询漏洞:")
try:
dangerous_results = dangerous_search_products(malicious_input)
print(f" 结果: {dangerous_results}")
except Exception as e:
print(f" 错误: {e}")
# 验证表仍然存在
cursor.execute("SELECT COUNT(*) FROM products")
count = cursor.fetchone()[0]
print(f"🛡️ 产品表保护成功，记录数: {count}")
def data_validation_and_sanitization(self):
"""演示数据验证和清理"""
print("\n=== 数据验证和清理演示 ===")
def validate_email(email: str) -> bool:
"""验证邮箱格式"""
pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}$'
return re.match(pattern, email) is not None
def validate_username(username: str) -> bool:
"""验证用户名格式"""
# 用户名：3-50个字符，只允许字母、数字、下划线
if len(username) < 3 or len(username) > 50:
return False
return re.match(r'^[a-zA-Z0-9_]+$', username) is not None
def sanitize_string(text: str) -> str:
"""清理字符串输入"""
# 移除潜在的恶意字符
# 保留基本字符，移除控制字符和特殊SQL字符
sanitized = re.sub(r'[\x00-\x1f\x7f-\x9f]', '', text)
sanitized = sanitized.replace('\0', '') # 移除空字节
return sanitized.strip()
def validate_price(price: Any) -> Optional[float]:
"""验证价格"""
try:
price_float = float(price)
if price_float >= 0 and price_float <= 999999.99:
return price_float
return None
except (ValueError, TypeError):
return None
# 测试数据
test_data = [
# (username, email, price, expected_result)
("valid_user", "[email protected]", 99.99, True),
("invalid user", "invalid-email", -10, False), # 多个问题
("[email protected]", "[email protected]", 0, False), # 用户名包含@
("123", "[email protected]", 1000000, False), # 价格过高
("", "[email protected]", 50, False), # 用户名太短
("user" * 20, "[email protected]", 25, False), # 用户名太长
]
cursor = self.connection.cursor()
valid_count = 0
for username, email, price, expected_valid in test_data:
print(f"\n测试数据: 用户名='{username}', 邮箱='{email}', 价格={price}")
# 验证各字段
username_valid = validate_username(username)
email_valid = validate_email(email)
price_valid = validate_price(price)
print(f" 用户名验证: {username_valid}")
print(f" 邮箱验证: {email_valid}")
print(f" 价格验证: {price_valid is not None}")
# 数据清理
clean_username = sanitize_string(username) if username_valid else None
clean_email = sanitize_string(email) if email_valid else None
clean_price = price_valid
print(f" 清理后用户名: {clean_username}")
print(f" 清理后邮箱: {clean_email}")
print(f" 清理后价格: {clean_price}")
# 综合验证
all_valid = all([username_valid, email_valid, price_valid is not None])
if all_valid:
try:
# 安全插入验证通过的数据
cursor.execute('''
INSERT INTO products (name, price, category)
VALUES (?, ?, ?)
''', (f"产品_{clean_username}", clean_price, "测试类别"))
valid_count += 1
print(f" ✅ 数据验证通过，已插入数据库")
except Exception as e:
print(f" ❌ 数据库插入失败: {e}")
else:
print(f" ❌ 数据验证失败，拒绝处理")
self.connection.commit()
print(f"\n📊 验证统计: {valid_count}/{len(test_data)} 条数据通过验证")
def performance_optimization_demo(self):
"""演示性能优化"""
print("\n=== 性能优化演示 ===")
cursor = self.connection.cursor()
# 创建大表用于性能测试
cursor.execute('''
CREATE TABLE IF NOT EXISTS performance_data (
id INTEGER PRIMARY KEY,
category INTEGER NOT NULL,
value REAL NOT NULL,
data TEXT,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建适当的索引
cursor.execute('CREATE INDEX IF NOT EXISTS idx_performance_category ON performance_data(category)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_performance_value ON performance_data(value)')
cursor.execute('CREATE INDEX IF NOT EXISTS idx_performance_timestamp ON performance_data(timestamp)')
self.connection.commit()
print("📋 性能测试表创建完成")
# 生成大量测试数据
data_size = 50000
print(f"🔄 生成 {data_size} 条测试数据...")
start_time = time.time()
# 批量插入优化
batch_size = 5000
for i in range(0, data_size, batch_size):
batch_data = []
for j in range(batch_size):
if i + j < data_size:
category = (i + j) % 10
value = float((i + j) * 0.1)
data = f"数据_{i+j}"
batch_data.append((category, value, data))
cursor.executemany('''
INSERT INTO performance_data (category, value, data)
VALUES (?, ?, ?)
''', batch_data)
if (i + batch_size) % 10000 == 0:
self.connection.commit()
print(f" 已插入 {i + batch_size} 条记录...")
self.connection.commit()
insert_time = time.time() - start_time
print(f"✅ 数据插入完成，耗时: {insert_time:.2f}秒")
# 查询性能测试
print("\n🔍 查询性能测试:")
# 测试1：简单查询
start_time = time.time()
cursor.execute('SELECT COUNT(*) FROM performance_data')
count = cursor.fetchone()[0]
simple_time = time.time() - start_time
print(f" 简单计数查询: {simple_time:.4f}秒，记录数: {count}")
# 测试2：条件查询（使用索引）
start_time = time.time()
cursor.execute('''
SELECT category, COUNT(*) as count, AVG(value) as avg_value
FROM performance_data
WHERE category BETWEEN 2 AND 7
GROUP BY category
ORDER BY count DESC
''')
results = cursor.fetchall()
indexed_time = time.time() - start_time
print(f" 索引条件查询: {indexed_time:.4f}秒，结果数: {len(results)}")
# 测试3：复杂聚合查询
start_time = time.time()
cursor.execute('''
SELECT
category,
MIN(value) as min_val,
MAX(value) as max_val,
AVG(value) as avg_val,
COUNT(*) as count
FROM performance_data
WHERE value > 100
GROUP BY category
HAVING COUNT(*) > 1000
ORDER BY avg_val DESC
''')
complex_results = cursor.fetchall()
complex_time = time.time() - start_time
print(f" 复杂聚合查询: {complex_time:.4f}秒，结果数: {len(complex_results)}")
# 查询执行计划分析
print("\n📈 查询执行计划分析:")
queries = [
"SELECT * FROM performance_data WHERE category = 5",
"SELECT * FROM performance_data WHERE value > 1000",
"SELECT category, AVG(value) FROM performance_data GROUP BY category"
]
for query in queries:
cursor.execute(f"EXPLAIN QUERY PLAN {query}")
plan = cursor.fetchall()
print(f" 查询: {query}")
for step in plan:
print(f" {step}")
print()
# 内存使用优化
print("💾 内存使用优化:")
# 清理测试数据
cursor.execute('DELETE FROM performance_data')
self.connection.commit()
# VACUUM操作优化数据库文件
start_time = time.time()
cursor.execute('VACUUM')
vacuum_time = time.time() - start_time
print(f" 数据库VACUUM优化: {vacuum_time:.2f}秒")
# 分析统计信息
cursor.execute('ANALYZE')
print(" 数据库统计信息已更新")
print("✅ 性能优化演示完成")
@contextmanager
def audit_context(self, user_id: Optional[int], action: str):
"""审计上下文管理器"""
start_time = time.time()
try:
yield
duration = time.time() - start_time
# 记录成功的操作
cursor = self.connection.cursor()
cursor.execute('''
INSERT INTO audit_log (user_id, action, timestamp)
VALUES (?, ?, CURRENT_TIMESTAMP)
''', (user_id, action))
self.connection.commit()
logger.info(f"Audit: User {user_id} performed {action} in {duration:.4f}s")
except Exception as e:
duration = time.time() - start_time
# 记录失败的操作
cursor = self.connection.cursor()
cursor.execute('''
INSERT INTO audit_log (user_id, action, new_values)
VALUES (?, ?, ?)
''', (user_id, action, f"ERROR: {str(e)}"))
self.connection.commit()
logger.error(f"Audit: User {user_id} failed {action} after {duration:.4f}s: {e}")
raise
def audit_and_logging_demo(self):
"""演示审计和日志记录"""
print("\n=== 审计和日志记录演示 ===")
cursor = self.connection.cursor()
# 获取测试用户ID
cursor.execute('SELECT id, username FROM users LIMIT 1')
user_data = cursor.fetchone()
if user_data:
user_id, username = user_data
print(f"使用测试用户: {username} (ID: {user_id})")
# 模拟一系列操作并记录审计日志
operations = [
"登录系统",
"查询产品",
"更新用户信息",
"删除数据",
"导出报告"
]
for operation in operations:
try:
with self.audit_context(user_id, operation):
# 模拟操作耗时
time.sleep(0.1)
# 模拟一些操作可能失败
if operation == "删除数据":
raise Exception("权限不足")
print(f" ✅ {operation} - 成功")
except Exception as e:
print(f" ❌ {operation} - 失败: {e}")
# 查询审计日志
cursor.execute('''
SELECT action, timestamp,
CASE WHEN new_values LIKE 'ERROR:%' THEN '失败' ELSE '成功' END as status
FROM audit_log
WHERE user_id = ?
ORDER BY timestamp DESC
''', (user_id,))
audit_records = cursor.fetchall()
print(f"\n📋 用户 {username} 的操作审计记录:")
for action, timestamp, status in audit_records:
print(f" {timestamp} - {action} - {status}")
else:
print("❌ 未找到测试用户")
def cleanup(self):
"""清理资源"""
if self.connection:
self.connection.close()
logger.info("🔒 数据库连接已关闭")
@staticmethod
def cleanup_database():
"""清理演示数据库文件"""
import os
db_files = ["secure_app.db", "secure_app.db-wal", "secure_app.db-shm"]
for file in db_files:
if os.path.exists(file):
os.remove(file)
print(f"🗑️ 已删除: {file}")
if __name__ == "__main__":
# 创建最佳实践演示实例
demo = SQLiteBestPractices()
try:
# 演示安全密码处理
demo.secure_password_handling()
# 演示SQL注入防护
demo.sql_injection_prevention()
# 演示数据验证和清理
demo.data_validation_and_sanitization()
# 演示性能优化
demo.performance_optimization_demo()
# 演示审计和日志
demo.audit_and_logging_demo()
finally:
# 清理资源
demo.cleanup()
# 清理演示文件
SQLiteBestPractices.cleanup_database()
print("\n🎉 最佳实践演示完成！")
---
2.2 commit与rollback
01.事务提交机制
a.commit操作原理
a.提交流程
SQLite的commit操作将事务中的所有更改永久保存到数据库，包括将脏页写入数据文件、更新WAL文件、同步日志等关键步骤。
b.持久化保证
通过fsync系统调用确保数据真正写入磁盘，提供ACID特性中的持久性保证，即使系统崩溃也不会丢失已提交的数据。
c.锁释放机制
提交成功后自动释放事务持有的所有锁，允许其他等待的事务继续执行，恢复系统的并发处理能力。
b.提交模式选择
a.同步提交Synchronous
PRAGMA synchronous控制提交时磁盘同步的级别，FULL提供最强持久性保证，NORMAL平衡性能和安全性，OFF提供最高性能但风险较大。
b.WAL模式提交
Write-Ahead Logging模式下，提交操作只需将变更写入WAL文件，大大提高了并发性能，允许读写操作同时进行。
c.批量提交优化
将多个相关操作组合在一个事务中批量提交，显著减少磁盘I/O次数，提升大批量数据操作的性能。
c.代码示例
---
# commit操作完整演示
import sqlite3
import time
import threading
from contextlib import contextmanager
from typing import List, Tuple, Optional
class CommitDemo:
def __init__(self, db_path: str = "commit_demo.db"):
"""初始化提交演示类"""
self.db_path = db_path
self.connection = None
self.setup_database()
def setup_database(self):
"""设置数据库连接和表结构"""
self.connection = sqlite3.connect(self.db_path)
cursor = self.connection.cursor()
# 启用外键约束
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
# 创建银行账户表
cursor.execute('''
CREATE TABLE IF NOT EXISTS accounts (
id INTEGER PRIMARY KEY AUTOINCREMENT,
account_number TEXT UNIQUE NOT NULL,
account_holder TEXT NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建交易记录表
cursor.execute('''
CREATE TABLE IF NOT EXISTS transactions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
from_account TEXT NOT NULL,
to_account TEXT NOT NULL,
amount REAL NOT NULL CHECK(amount > 0),
transaction_type TEXT NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
completed_at TIMESTAMP,
FOREIGN KEY (from_account) REFERENCES accounts(account_number),
FOREIGN KEY (to_account) REFERENCES accounts(account_number)
)
''')
# 创建审计日志表
cursor.execute('''
CREATE TABLE IF NOT EXISTS audit_log (
id INTEGER PRIMARY KEY AUTOINCREMENT,
table_name TEXT NOT NULL,
operation TEXT NOT NULL,
record_id INTEGER,
old_values TEXT,
new_values TEXT,
user_id TEXT,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
self.connection.commit()
print("✅ 数据库结构创建完成")
@contextmanager
def transaction_context(self, synchronous_mode: str = "NORMAL"):
"""事务上下文管理器"""
cursor = self.connection.cursor()
try:
# 设置同步模式
cursor.execute(f'PRAGMA synchronous = {synchronous_mode}')
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
yield cursor
# 提交事务
self.connection.commit()
print(f"✅ 事务提交成功 (同步模式: {synchronous_mode})")
except Exception as e:
# 回滚事务
self.connection.rollback()
print(f"❌ 事务回滚: {e}")
raise
def basic_commit_demo(self):
"""基础提交演示"""
print("=== 基础提交演示 ===")
with self.transaction_context() as cursor:
# 插入测试账户
accounts = [
('ACC001', '张三', 5000.0),
('ACC002', '李四', 3000.0),
('ACC003', '王五', 8000.0)
]
for acc_num, holder, balance in accounts:
cursor.execute('''
INSERT INTO accounts (account_number, account_holder, balance)
VALUES (?, ?, ?)
''', (acc_num, holder, balance))
print(f" 创建账户: {acc_num} - {holder} - ¥{balance}")
# 查询插入结果
cursor.execute('SELECT COUNT(*) FROM accounts')
count = cursor.fetchone()[0]
print(f" 总账户数: {count}")
def transfer_operation(self, from_account: str, to_account: str, amount: float):
"""执行转账操作"""
with self.transaction_context() as cursor:
# 验证账户存在
cursor.execute('SELECT balance FROM accounts WHERE account_number = ?', (from_account,))
from_result = cursor.fetchone()
if not from_result:
raise ValueError(f"源账户不存在: {from_account}")
cursor.execute('SELECT balance FROM accounts WHERE account_number = ?', (to_account,))
to_result = cursor.fetchone()
if not to_result:
raise ValueError(f"目标账户不存在: {to_account}")
# 检查余额
from_balance = from_result[0]
if from_balance < amount:
raise ValueError(f"余额不足: {from_account} 余额 ¥{from_balance}, 转账金额 ¥{amount}")
# 扣款
cursor.execute('''
UPDATE accounts
SET balance = balance - ?
WHERE account_number = ?
''', (amount, from_account))
print(f" {from_account} 扣除 ¥{amount}")
# 入账
cursor.execute('''
UPDATE accounts
SET balance = balance + ?
WHERE account_number = ?
''', (amount, to_account))
print(f" {to_account} 增加 ¥{amount}")
# 记录交易
cursor.execute('''
INSERT INTO transactions (from_account, to_account, amount, transaction_type, status)
VALUES (?, ?, ?, ?, ?)
''', (from_account, to_account, amount, 'TRANSFER', 'completed'))
print(f" ✅ 转账完成: {from_account} → {to_account} ¥{amount}")
def synchronous_mode_comparison(self):
"""不同同步模式性能对比"""
print("\n=== 同步模式性能对比 ===")
# 准备测试数据
test_records = 1000
test_data = [
(f'TEST{i:04d}', f'测试用户{i}', float(i * 10))
for i in range(1, test_records + 1)
]
modes = ['OFF', 'NORMAL', 'FULL']
results = []
for mode in modes:
print(f"\n测试同步模式: {mode}")
# 清空表
cursor = self.connection.cursor()
cursor.execute('DELETE FROM accounts')
self.connection.commit()
# 性能测试
start_time = time.time()
with self.transaction_context(mode) as cursor:
for acc_num, holder, balance in test_data:
cursor.execute('''
INSERT INTO accounts (account_number, account_holder, balance)
VALUES (?, ?, ?)
''', (acc_num, holder, balance))
end_time = time.time()
duration = end_time - start_time
# 验证数据完整性
cursor.execute('SELECT COUNT(*) FROM accounts')
count = cursor.fetchone()[0]
results.append({
'mode': mode,
'duration': duration,
'records': count,
'records_per_sec': count / duration if duration > 0 else 0
})
print(f" 耗时: {duration:.4f}秒")
print(f" 记录数: {count}")
print(f" 速率: {count/duration:.0f} 记录/秒")
# 显示对比结果
print("\n性能对比结果:")
print(f"{'模式':<8} {'耗时(秒)':<10} {'记录数':<8} {'速率(条/秒)'}")
print("-" * 40)
for result in results:
print(f"{result['mode']:<8} {result['duration']:<10.4f} {result['records']:<8} {result['records_per_sec']:<8.0f}")
def concurrent_commit_demo(self):
"""并发提交演示"""
print("\n=== 并发提交演示 ===")
# 清空并重置数据
cursor = self.connection.cursor()
cursor.execute('DELETE FROM accounts')
cursor.execute('DELETE FROM transactions')
self.connection.commit()
# 创建初始账户
initial_accounts = [
('BANK001', '主账户', 100000.0),
('USER001', '用户A', 1000.0),
('USER002', '用户B', 1000.0),
('USER003', '用户C', 1000.0)
]
with self.transaction_context() as cursor:
for acc_num, holder, balance in initial_accounts:
cursor.execute('''
INSERT INTO accounts (account_number, account_holder, balance)
VALUES (?, ?, ?)
''', (acc_num, holder, balance))
# 模拟并发转账操作
def transfer_worker(worker_id: int, transfers: List[Tuple]):
"""转账工作线程"""
try:
# 每个线程使用独立连接
conn = sqlite3.connect(self.db_path)
conn.execute('PRAGMA foreign_keys = ON')
conn.execute('PRAGMA journal_mode = WAL')
for from_acc, to_acc, amount in transfers:
try:
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 检查余额
cursor.execute('SELECT balance FROM accounts WHERE account_number = ?', (from_acc,))
result = cursor.fetchone()
if not result or result[0] < amount:
conn.rollback()
print(f" 工作线程{worker_id}: 转账失败 - 余额不足 {from_acc}")
continue
# 执行转账
cursor.execute('UPDATE accounts SET balance = balance - ? WHERE account_number = ?', (amount, from_acc))
cursor.execute('UPDATE accounts SET balance = balance + ? WHERE account_number = ?', (amount, to_acc))
# 记录交易
cursor.execute('''
INSERT INTO transactions (from_account, to_account, amount, transaction_type, status)
VALUES (?, ?, ?, ?, ?)
''', (from_acc, to_acc, amount, 'CONCURRENT_TRANSFER', 'completed'))
conn.commit()
print(f" 工作线程{worker_id}: 转账成功 {from_acc} → {to_acc} ¥{amount}")
# 短暂休眠模拟真实场景
time.sleep(0.01)
except Exception as e:
conn.rollback()
print(f" 工作线程{worker_id}: 转账异常 {e}")
conn.close()
except Exception as e:
print(f"工作线程{worker_id} 异常: {e}")
# 准备转账数据
transfer_plans = [
[('BANK001', 'USER001', 500.0), ('BANK001', 'USER002', 300.0), ('BANK001', 'USER003', 200.0)],
[('USER001', 'USER002', 100.0), ('USER001', 'USER003', 150.0)],
[('USER002', 'USER001', 50.0), ('USER002', 'USER003', 100.0)],
[('USER003', 'USER001', 80.0), ('USER003', 'USER002', 120.0)]
]
# 启动并发线程
threads = []
for i, transfers in enumerate(transfer_plans):
thread = threading.Thread(target=transfer_worker, args=(i+1, transfers))
threads.append(thread)
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 验证最终结果
print("\n最终账户余额:")
cursor = self.connection.cursor()
cursor.execute('SELECT account_number, account_holder, balance FROM accounts ORDER BY account_number')
accounts = cursor.fetchall()
for acc_num, holder, balance in accounts:
print(f" {acc_num}: {holder} - ¥{balance:.2f}")
# 验证交易记录
cursor.execute('SELECT COUNT(*) FROM transactions WHERE status = "completed"')
transaction_count = cursor.fetchone()[0]
print(f"\n总交易数: {transaction_count}")
def cleanup(self):
"""清理资源"""
if self.connection:
self.connection.close()
print("🔒 数据库连接已关闭")
if __name__ == "__main__":
demo = CommitDemo("commit_demo.db")
try:
# 基础提交演示
demo.basic_commit_demo()
# 转账操作演示
print("\n=== 转账操作演示 ===")
demo.transfer_operation('ACC001', 'ACC002', 1000.0)
demo.transfer_operation('ACC002', 'ACC003', 500.0)
# 同步模式对比
demo.synchronous_mode_comparison()
# 并发提交演示
demo.concurrent_commit_demo()
finally:
demo.cleanup()
# 清理测试文件
import os
files_to_remove = ['commit_demo.db', 'commit_demo.db-wal', 'commit_demo.db-shm']
for file in files_to_remove:
if os.path.exists(file):
os.remove(file)
print(f"🗑️ 已删除: {file}")
---
02.事务回滚机制
a.rollback操作原理
a.回滚触发条件
显式调用ROLLBACK语句、异常处理中的自动回滚、连接中断导致的系统回滚，以及约束违反时的强制回滚等。
b.Undo日志机制
SQLite通过维护Undo日志记录事务执行前的数据状态，回滚时逆向执行撤销操作，将数据恢复到事务开始前的一致状态。
c.资源清理过程
回滚过程中释放事务占用的所有锁资源，清除临时数据结构，撤销所有未提交的数据修改。
b.回滚策略优化
a.保存点机制
在长事务中设置多个保存点，允许部分回滚而不是完全回滚，提高复杂业务操作的灵活性和性能。
b.延迟验证策略
将复杂的数据验证操作放在事务后期执行，减少因早期验证失败导致的频繁回滚开销。
c.批量操作回滚
对于批量数据处理，采用分段事务策略，避免单次回滚操作影响大量数据，提高系统响应性。
c.代码示例
---
# rollback操作完整演示
import sqlite3
import time
import random
from contextlib import contextmanager
from typing import List, Dict, Any
class RollbackDemo:
def __init__(self, db_path: str = "rollback_demo.db"):
"""初始化回滚演示类"""
self.db_path = db_path
self.connection = None
self.setup_database()
def setup_database(self):
"""设置数据库和表结构"""
self.connection = sqlite3.connect(self.db_path)
cursor = self.connection.cursor()
# 启用外键约束和WAL模式
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
# 创建订单表
cursor.execute('''
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
order_number TEXT UNIQUE NOT NULL,
customer_name TEXT NOT NULL,
total_amount REAL NOT NULL CHECK(total_amount > 0),
status TEXT NOT NULL DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 创建订单项表
cursor.execute('''
CREATE TABLE IF NOT EXISTS order_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
order_id INTEGER NOT NULL,
product_name TEXT NOT NULL,
quantity INTEGER NOT NULL CHECK(quantity > 0),
unit_price REAL NOT NULL CHECK(unit_price > 0),
total_price REAL NOT NULL CHECK(total_price > 0),
FOREIGN KEY (order_id) REFERENCES orders(id) ON DELETE CASCADE
)
''')
# 创建库存表
cursor.execute('''
CREATE TABLE IF NOT EXISTS inventory (
id INTEGER PRIMARY KEY AUTOINCREMENT,
product_name TEXT UNIQUE NOT NULL,
stock_quantity INTEGER NOT NULL CHECK(stock_quantity >= 0),
unit_price REAL NOT NULL CHECK(unit_price > 0),
last_updated TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 初始化库存数据
initial_inventory = [
('笔记本电脑', 50, 5999.99),
('无线鼠标', 200, 99.99),
('机械键盘', 100, 299.99),
('显示器', 80, 1599.99),
('USB集线器', 300, 49.99)
]
cursor.executemany('''
INSERT OR REPLACE INTO inventory (product_name, stock_quantity, unit_price)
VALUES (?, ?, ?)
''', initial_inventory)
self.connection.commit()
print("✅ 数据库结构创建完成")
@contextmanager
def order_transaction_context(self, order_number: str):
"""订单事务上下文管理器"""
cursor = self.connection.cursor()
try:
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print(f"🚀 开始订单事务: {order_number}")
yield cursor
self.connection.commit()
print(f"✅ 订单事务提交成功: {order_number}")
except Exception as e:
self.connection.rollback()
print(f"❌ 订单事务回滚: {order_number} - {e}")
raise
def successful_order_demo(self):
"""成功订单演示"""
print("\n=== 成功订单演示 ===")
with self.order_transaction_context("ORD-001") as cursor:
# 创建订单
cursor.execute('''
INSERT INTO orders (order_number, customer_name, total_amount, status)
VALUES (?, ?, ?, ?)
''', ("ORD-001", "张三", 0.0, "processing"))
order_id = cursor.lastrowid
print(f" 创建订单: ORD-001 (ID: {order_id})")
# 订单项
order_items = [
("笔记本电脑", 1, 5999.99),
("无线鼠标", 2, 99.99),
("机械键盘", 1, 299.99)
]
total_amount = 0
for product, quantity, unit_price in order_items:
# 检查库存
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"库存不足: {product}, 当前库存: {stock}, 需求: {quantity}")
# 扣减库存
cursor.execute('''
UPDATE inventory
SET stock_quantity = stock_quantity - ?, last_updated = CURRENT_TIMESTAMP
WHERE product_name = ?
''', (quantity, product))
print(f" 扣减库存: {product} x{quantity}")
# 添加订单项
item_total = quantity * unit_price
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product, quantity, unit_price, item_total))
total_amount += item_total
# 更新订单总金额
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (total_amount, order_id))
print(f" 订单确认，总金额: ¥{total_amount:.2f}")
def inventory_insufficient_rollback_demo(self):
"""库存不足回滚演示"""
print("\n=== 库存不足回滚演示 ===")
with self.order_transaction_context("ORD-002") as cursor:
# 创建订单
cursor.execute('''
INSERT INTO orders (order_number, customer_name, total_amount, status)
VALUES (?, ?, ?, ?)
''', ("ORD-002", "李四", 0.0, "processing"))
order_id = cursor.lastrowid
print(f" 创建订单: ORD-002 (ID: {order_id})")
# 尝试购买超出库存的商品
order_items = [
("笔记本电脑", 1, 5999.99), # 正常
("显示器", 100, 1599.99), # 超出库存（只有80个）
("无线鼠标", 1, 99.99) # 正常
]
total_amount = 0
for product, quantity, unit_price in order_items:
# 检查库存
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"库存不足: {product}, 当前库存: {stock}, 需求: {quantity}")
# 扣减库存（这里不会执行到，因为上面会抛出异常）
cursor.execute('''
UPDATE inventory
SET stock_quantity = stock_quantity - ?, last_updated = CURRENT_TIMESTAMP
WHERE product_name = ?
''', (quantity, product))
item_total = quantity * unit_price
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product, quantity, unit_price, item_total))
total_amount += item_total
def constraint_violation_rollback_demo(self):
"""约束违反回滚演示"""
print("\n=== 约束违反回滚演示 ===")
with self.order_transaction_context("ORD-003") as cursor:
# 创建订单
cursor.execute('''
INSERT INTO orders (order_number, customer_name, total_amount, status)
VALUES (?, ?, ?, ?)
''', ("ORD-003", "王五", 1000.0, "processing"))
order_id = cursor.lastrowid
print(f" 创建订单: ORD-003 (ID: {order_id})")
# 尝试插入违反约束的订单项（负价格）
try:
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, "违规商品", 1, -100.0, -100.0))
print(" 插入违规订单项")
except sqlite3.IntegrityError as e:
print(f" ❌ 约束违反: {e}")
# 这里的异常会被上下文管理器捕获并触发回滚
def savepoint_rollback_demo(self):
"""保存点回滚演示"""
print("\n=== 保存点回滚演示 ===")
cursor = self.connection.cursor()
try:
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print("🚀 开始复杂订单事务: ORD-004")
# 创建订单
cursor.execute('''
INSERT INTO orders (order_number, customer_name, total_amount, status)
VALUES (?, ?, ?, ?)
''', ("ORD-004", "赵六", 0.0, "processing"))
order_id = cursor.lastrowid
print(f" 创建订单: ORD-004 (ID: {order_id})")
# 设置第一个保存点
cursor.execute('SAVEPOINT sp_order_created')
print(" 📍 设置保存点: sp_order_created")
# 第一批商品处理
batch1_items = [
("笔记本电脑", 1, 5999.99),
("无线鼠标", 1, 99.99)
]
batch1_total = 0
for product, quantity, unit_price in batch1_items:
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"第一批商品库存不足: {product}")
cursor.execute('''
UPDATE inventory
SET stock_quantity = stock_quantity - ?, last_updated = CURRENT_TIMESTAMP
WHERE product_name = ?
''', (quantity, product))
item_total = quantity * unit_price
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product, quantity, unit_price, item_total))
batch1_total += item_total
print(f" 处理商品: {product} x{quantity}")
# 设置第二个保存点
cursor.execute('SAVEPOINT sp_batch1_complete')
print(" 📍 设置保存点: sp_batch1_complete")
# 尝试处理第二批商品（可能失败）
try:
batch2_items = [
("机械键盘", 1, 299.99),
("显示器", 200, 1599.99) # 故意设置超出库存的数量
]
batch2_total = 0
for product, quantity, unit_price in batch2_items:
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"第二批商品库存不足: {product}")
cursor.execute('''
UPDATE inventory
SET stock_quantity = stock_quantity - ?, last_updated = CURRENT_TIMESTAMP
WHERE product_name = ?
''', (quantity, product))
item_total = quantity * unit_price
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product, quantity, unit_price, item_total))
batch2_total += item_total
# 设置第三个保存点（这里不会到达）
cursor.execute('SAVEPOINT sp_batch2_complete')
except ValueError as e:
print(f" ⚠️ 第二批商品处理失败: {e}")
print(" ↩️ 回滚到保存点: sp_batch1_complete")
cursor.execute('ROLLBACK TO SAVEPOINT sp_batch1_complete')
# 不使用第二批商品，继续处理
batch2_total = 0
# 更新订单总金额（只包含第一批商品）
final_total = batch1_total + batch2_total
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (final_total, order_id))
self.connection.commit()
print(f"✅ 复杂订单事务提交成功，总金额: ¥{final_total:.2f}")
except Exception as e:
self.connection.rollback()
print(f"❌ 复杂订单事务回滚: {e}")
def batch_rollback_demo(self):
"""批量回滚演示"""
print("\n=== 批量回滚演示 ===")
# 模拟批量订单处理
batch_orders = [
("ORD-BATCH-001", "客户A", [("无线鼠标", 5, 99.99), ("USB集线器", 10, 49.99)]),
("ORD-BATCH-002", "客户B", [("机械键盘", 3, 299.99), ("显示器", 2, 1599.99)]),
("ORD-BATCH-003", "客户C", [("笔记本电脑", 2, 5999.99)]), # 库存不足
("ORD-BATCH-004", "客户D", [("无线鼠标", 3, 99.99)])
]
successful_orders = 0
failed_orders = 0
for order_num, customer, items in batch_orders:
try:
with self.order_transaction_context(order_num) as cursor:
# 创建订单
cursor.execute('''
INSERT INTO orders (order_number, customer_name, total_amount, status)
VALUES (?, ?, ?, ?)
''', (order_num, customer, 0.0, "processing"))
order_id = cursor.lastrowid
total_amount = 0
for product, quantity, unit_price in items:
# 检查库存
cursor.execute('SELECT stock_quantity FROM inventory WHERE product_name = ?', (product,))
stock = cursor.fetchone()[0]
if stock < quantity:
raise ValueError(f"库存不足: {product}")
# 扣减库存
cursor.execute('''
UPDATE inventory
SET stock_quantity = stock_quantity - ?, last_updated = CURRENT_TIMESTAMP
WHERE product_name = ?
''', (quantity, product))
item_total = quantity * unit_price
cursor.execute('''
INSERT INTO order_items (order_id, product_name, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product, quantity, unit_price, item_total))
total_amount += item_total
# 更新订单
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (total_amount, order_id))
print(f" ✅ {order_num}: 总金额 ¥{total_amount:.2f}")
successful_orders += 1
except Exception as e:
print(f" ❌ {order_num}: {e}")
failed_orders += 1
print(f"\n批量处理结果: 成功 {successful_orders}，失败 {failed_orders}")
def check_final_state(self):
"""检查最终状态"""
print("\n=== 最终状态检查 ===")
cursor = self.connection.cursor()
# 检查订单状态
cursor.execute('SELECT order_number, customer_name, total_amount, status FROM orders ORDER BY created_at')
orders = cursor.fetchall()
print("订单列表:")
for order_num, customer, amount, status in orders:
print(f" {order_num}: {customer} - ¥{amount:.2f} - {status}")
# 检查库存状态
cursor.execute('SELECT product_name, stock_quantity FROM inventory ORDER BY product_name')
inventory = cursor.fetchall()
print("\n库存状态:")
for product, stock in inventory:
print(f" {product}: {stock}")
# 检查订单项总数
cursor.execute('SELECT COUNT(*) FROM order_items')
item_count = cursor.fetchone()[0]
print(f"\n订单项总数: {item_count}")
def cleanup(self):
"""清理资源"""
if self.connection:
self.connection.close()
print("🔒 数据库连接已关闭")
if __name__ == "__main__":
demo = RollbackDemo("rollback_demo.db")
try:
# 成功订单演示
demo.successful_order_demo()
# 库存不足回滚演示
demo.inventory_insufficient_rollback_demo()
# 约束违反回滚演示
demo.constraint_violation_rollback_demo()
# 保存点回滚演示
demo.savepoint_rollback_demo()
# 批量回滚演示
demo.batch_rollback_demo()
# 检查最终状态
demo.check_final_state()
finally:
demo.cleanup()
# 清理测试文件
import os
files_to_remove = ['rollback_demo.db', 'rollback_demo.db-wal', 'rollback_demo.db-shm']
for file in files_to_remove:
if os.path.exists(file):
os.remove(file)
print(f"🗑️ 已删除: {file}")
---
03.提交与回滚的最佳实践
a.事务设计原则
a.短事务原则
保持事务尽可能简短，只在必要时包含操作，减少锁持有时间，提高系统并发性能和响应能力。
b.原子性设计
将相关的操作组合在一个事务中，确保这些操作要么全部成功，要么全部失败，维护数据的一致性。
c.错误处理策略
设计完善的异常处理机制，确保在任何错误情况下都能正确回滚事务，避免数据不一致状态。
b.性能优化建议
a.批量操作优化
对于大量数据操作，采用分批处理策略，避免单次事务处理过多数据导致性能问题。
b.索引优化
合理设计索引结构，减少事务执行时间，特别是在涉及大量数据查询和更新操作时。
c.连接池管理
使用连接池技术管理数据库连接，避免频繁创建和销毁连接带来的性能开销。
c.代码示例
---
# 提交与回滚最佳实践演示
import sqlite3
import time
import logging
from contextlib import contextmanager
from typing import Optional, List, Dict, Any, Callable
from dataclasses import dataclass
from enum import Enum
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class TransactionStatus(Enum):
"""事务状态枚举"""
PENDING = "pending"
COMMITTED = "committed"
ROLLED_BACK = "rolled_back"
@dataclass
class TransactionMetrics:
"""事务指标数据类"""
transaction_id: str
start_time: float
end_time: Optional[float] = None
status: TransactionStatus = TransactionStatus.PENDING
operations_count: int = 0
rollback_reason: Optional[str] = None
class TransactionBestPractices:
def __init__(self, db_path: str = "best_practices_demo.db"):
"""初始化最佳实践演示类"""
self.db_path = db_path
self.connection = None
self.transaction_metrics: Dict[str, TransactionMetrics] = {}
self.setup_database()
def setup_database(self):
"""设置数据库结构"""
try:
self.connection = sqlite3.connect(
self.db_path,
timeout=30.0,
check_same_thread=False
)
cursor = self.connection.cursor()
# 优化配置
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
cursor.execute('PRAGMA synchronous = NORMAL')
cursor.execute('PRAGMA cache_size = 10000')
cursor.execute('PRAGMA temp_store = MEMORY')
# 创建业务表
self._create_business_tables(cursor)
# 创建监控表
self._create_monitoring_tables(cursor)
self.connection.commit()
logger.info("✅ 数据库初始化完成")
except Exception as e:
logger.error(f"❌ 数据库初始化失败: {e}")
raise
def _create_business_tables(self, cursor):
"""创建业务表结构"""
# 用户表
cursor.execute('''
CREATE TABLE IF NOT EXISTS users (
id INTEGER PRIMARY KEY AUTOINCREMENT,
username TEXT UNIQUE NOT NULL,
email TEXT UNIQUE NOT NULL,
balance REAL NOT NULL DEFAULT 0.0 CHECK(balance >= 0),
status TEXT NOT NULL DEFAULT 'active',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 产品表
cursor.execute('''
CREATE TABLE IF NOT EXISTS products (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
price REAL NOT NULL CHECK(price > 0),
stock_quantity INTEGER NOT NULL DEFAULT 0 CHECK(stock_quantity >= 0),
category TEXT NOT NULL,
is_active BOOLEAN DEFAULT 1,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
# 订单表
cursor.execute('''
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
order_number TEXT UNIQUE NOT NULL,
total_amount REAL NOT NULL CHECK(total_amount >= 0),
status TEXT NOT NULL DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (user_id) REFERENCES users(id)
)
''')
# 订单项表
cursor.execute('''
CREATE TABLE IF NOT EXISTS order_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
order_id INTEGER NOT NULL,
product_id INTEGER NOT NULL,
quantity INTEGER NOT NULL CHECK(quantity > 0),
unit_price REAL NOT NULL CHECK(unit_price > 0),
total_price REAL NOT NULL CHECK(total_price > 0),
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (order_id) REFERENCES orders(id),
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
logger.info("📋 业务表结构创建完成")
def _create_monitoring_tables(self, cursor):
"""创建监控表结构"""
# 事务监控表
cursor.execute('''
CREATE TABLE IF NOT EXISTS transaction_monitor (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transaction_id TEXT NOT NULL,
operation_type TEXT NOT NULL,
table_name TEXT,
start_time TIMESTAMP NOT NULL,
end_time TIMESTAMP,
status TEXT NOT NULL,
operations_count INTEGER DEFAULT 0,
rollback_reason TEXT,
duration_ms REAL
)
''')
# 性能监控表
cursor.execute('''
CREATE TABLE IF NOT EXISTS performance_metrics (
id INTEGER PRIMARY KEY AUTOINCREMENT,
metric_name TEXT NOT NULL,
metric_value REAL NOT NULL,
unit TEXT NOT NULL,
timestamp TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
logger.info("📊 监控表结构创建完成")
@contextmanager
def monitored_transaction(self, transaction_id: str, auto_commit: bool = False):
"""带监控的事务上下文管理器"""
cursor = self.connection.cursor()
start_time = time.time()
# 记录事务开始
metrics = TransactionMetrics(
transaction_id=transaction_id,
start_time=start_time
)
self.transaction_metrics[transaction_id] = metrics
try:
if not auto_commit:
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
logger.info(f"🚀 事务开始: {transaction_id}")
yield cursor
if not auto_commit:
self.connection.commit()
# 更新成功指标
metrics.end_time = time.time()
metrics.status = TransactionStatus.COMMITTED
# 记录到数据库
cursor.execute('''
INSERT INTO transaction_monitor
(transaction_id, operation_type, start_time, end_time, status, operations_count, duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
transaction_id, 'MONITORED_TRANSACTION',
start_time, metrics.end_time,
metrics.status.value, metrics.operations_count,
(metrics.end_time - start_time) * 1000
))
if auto_commit:
self.connection.commit()
logger.info(f"✅ 事务提交成功: {transaction_id} (耗时: {(metrics.end_time - start_time):.3f}秒)")
except Exception as e:
# 回滚处理
if not auto_commit:
self.connection.rollback()
# 更新失败指标
metrics.end_time = time.time()
metrics.status = TransactionStatus.ROLLED_BACK
metrics.rollback_reason = str(e)
# 记录到数据库
cursor.execute('''
INSERT INTO transaction_monitor
(transaction_id, operation_type, start_time, end_time, status, operations_count, rollback_reason, duration_ms)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)
''', (
transaction_id, 'MONITORED_TRANSACTION',
start_time, metrics.end_time,
metrics.status.value, metrics.operations_count,
str(e), (metrics.end_time - start_time) * 1000
))
if auto_commit:
self.connection.commit()
logger.error(f"❌ 事务回滚: {transaction_id} - {e}")
raise
def short_transaction_demo(self):
"""短事务原则演示"""
print("\n=== 短事务原则演示 ===")
# 初始化测试数据
with self.monitored_transaction("INIT_DATA") as cursor:
# 创建用户
cursor.executemany('''
INSERT INTO users (username, email, balance)
VALUES (?, ?, ?)
''', [
('alice', '[email protected]', 1000.0),
('bob', '[email protected]', 500.0),
('charlie', '[email protected]', 2000.0)
])
# 创建产品
cursor.executemany('''
INSERT INTO products (name, price, stock_quantity, category)
VALUES (?, ?, ?, ?)
''', [
('智能手机', 2999.99, 50, '电子产品'),
('笔记本电脑', 6999.99, 30, '电子产品'),
('无线耳机', 299.99, 100, '电子产品'),
('保护套', 49.99, 200, '配件')
])
# 演示短事务：简单转账
def simple_transfer(from_user: str, to_user: str, amount: float):
"""简单转账操作（短事务）"""
with self.monitored_transaction(f"TRANSFER_{from_user}_TO_{to_user}_{int(time.time())}") as cursor:
# 查询用户ID
cursor.execute('SELECT id, balance FROM users WHERE username = ?', (from_user,))
from_result = cursor.fetchone()
if not from_result or from_result[1] < amount:
raise ValueError(f"用户 {from_user} 余额不足")
cursor.execute('SELECT id FROM users WHERE username = ?', (to_user,))
to_result = cursor.fetchone()
if not to_result:
raise ValueError(f"目标用户 {to_user} 不存在")
# 执行转账
cursor.execute('UPDATE users SET balance = balance - ? WHERE id = ?', (amount, from_result[0]))
cursor.execute('UPDATE users SET balance = balance + ? WHERE id = ?', (amount, to_result[0]))
# 增加操作计数
transaction_id = f"TRANSFER_{from_user}_TO_{to_user}_{int(time.time())}"
if transaction_id in self.transaction_metrics:
self.transaction_metrics[transaction_id].operations_count = 2
# 执行转账
simple_transfer('alice', 'bob', 200.0)
simple_transfer('charlie', 'alice', 500.0)
# 查询最终余额
cursor = self.connection.cursor()
cursor.execute('SELECT username, balance FROM users ORDER BY username')
users = cursor.fetchall()
print("用户余额:")
for username, balance in users:
print(f" {username}: ¥{balance:.2f}")
def atomic_transaction_demo(self):
"""原子性事务演示"""
print("\n=== 原子性事务演示 ===")
def create_order_with_inventory_check(user_id: int, order_items: List[Dict]):
"""创建订单并检查库存（原子性操作）"""
order_number = f"ORD{int(time.time())}"
with self.monitored_transaction(f"ORDER_{order_number}") as cursor:
# 创建订单
cursor.execute('''
INSERT INTO orders (user_id, order_number, total_amount, status)
VALUES (?, ?, ?, ?)
''', (user_id, order_number, 0.0, 'processing'))
order_id = cursor.lastrowid
total_amount = 0.0
operations_count = 0
# 处理每个订单项
for item in order_items:
product_name = item['product']
quantity = item['quantity']
# 检查产品是否存在且有库存
cursor.execute('SELECT id, price, stock_quantity FROM products WHERE name = ? AND is_active = 1', (product_name,))
product = cursor.fetchone()
if not product:
raise ValueError(f"产品不存在或已下架: {product_name}")
product_id, price, stock = product
if stock < quantity:
raise ValueError(f"库存不足: {product_name}, 当前: {stock}, 需要: {quantity}")
# 扣减库存
cursor.execute('''
UPDATE products
SET stock_quantity = stock_quantity - ?
WHERE id = ?
''', (quantity, product_id))
operations_count += 1
# 创建订单项
item_total = price * quantity
cursor.execute('''
INSERT INTO order_items (order_id, product_id, quantity, unit_price, total_price)
VALUES (?, ?, ?, ?, ?)
''', (order_id, product_id, quantity, price, item_total))
operations_count += 1
total_amount += item_total
# 更新订单总金额
cursor.execute('''
UPDATE orders SET total_amount = ?, status = 'confirmed'
WHERE id = ?
''', (total_amount, order_id))
operations_count += 1
# 更新操作计数
transaction_id = f"ORDER_{order_number}"
if transaction_id in self.transaction_metrics:
self.transaction_metrics[transaction_id].operations_count = operations_count
return order_id
# 测试成功订单
try:
print("创建成功订单...")
successful_items = [
{'product': '智能手机', 'quantity': 1},
{'product': '无线耳机', 'quantity': 2}
]
order_id = create_order_with_inventory_check(1, successful_items)
print(f"✅ 订单创建成功: ID {order_id}")
except Exception as e:
print(f"❌ 订单创建失败: {e}")
# 测试失败订单（库存不足）
try:
print("\n创建库存不足的订单...")
failed_items = [
{'product': '笔记本电脑', 'quantity': 50} # 库存只有30
]
order_id = create_order_with_inventory_check(2, failed_items)
print(f"✅ 订单创建成功: ID {order_id}")
except Exception as e:
print(f"❌ 订单创建失败: {e}")
# 查询库存状态
cursor = self.connection.cursor()
cursor.execute('SELECT name, stock_quantity FROM products ORDER BY name')
products = cursor.fetchall()
print("\n产品库存:")
for name, stock in products:
print(f" {name}: {stock}")
def batch_operation_demo(self):
"""批量操作优化演示"""
print("\n=== 批量操作优化演示 ===")
def batch_update_prices(updates: List[Dict]):
"""批量更新价格"""
with self.monitored_transaction("BATCH_PRICE_UPDATE") as cursor:
operations_count = 0
for update in updates:
cursor.execute('''
UPDATE products
SET price = ?
WHERE name = ?
''', (update['new_price'], update['product_name']))
operations_count += 1
# 更新操作计数
if "BATCH_PRICE_UPDATE" in self.transaction_metrics:
self.transaction_metrics["BATCH_PRICE_UPDATE"].operations_count = operations_count
# 准备批量更新数据
price_updates = [
{'product_name': '智能手机', 'new_price': 2799.99},
{'product_name': '笔记本电脑', 'new_price': 6499.99},
{'product_name': '无线耳机', 'new_price': 249.99},
{'product_name': '保护套', 'new_price': 39.99}
]
# 执行批量更新
batch_update_prices(price_updates)
print("✅ 批量价格更新完成")
# 查询更新结果
cursor = self.connection.cursor()
cursor.execute('SELECT name, price FROM products ORDER BY name')
products = cursor.fetchall()
print("更新后的价格:")
for name, price in products:
print(f" {name}: ¥{price:.2f}")
def transaction_metrics_report(self):
"""事务指标报告"""
print("\n=== 事务指标报告 ===")
# 从数据库查询指标
cursor = self.connection.cursor()
cursor.execute('''
SELECT
transaction_id,
status,
operations_count,
duration_ms,
rollback_reason
FROM transaction_monitor
ORDER BY start_time DESC
''')
metrics = cursor.fetchall()
print("事务执行历史:")
for trans_id, status, ops_count, duration_ms, reason in metrics:
print(f" {trans_id}:")
print(f" 状态: {status}")
print(f" 操作数: {ops_count}")
print(f" 耗时: {duration_ms:.2f}ms")
if reason:
print(f" 回滚原因: {reason}")
# 统计汇总
cursor.execute('''
SELECT
status,
COUNT(*) as count,
AVG(duration_ms) as avg_duration,
SUM(operations_count) as total_operations
FROM transaction_monitor
GROUP BY status
''')
summary = cursor.fetchall()
print("\n统计汇总:")
for status, count, avg_duration, total_ops in summary:
print(f" {status}: {count}次, 平均耗时 {avg_duration:.2f}ms, 总操作数 {total_ops}")
def performance_optimization_demo(self):
"""性能优化演示"""
print("\n=== 性能优化演示 ===")
# 测试不同事务大小的性能
def performance_test(batch_size: int, description: str):
"""性能测试"""
test_data = [
(f'TEST_USER_{i}', f'user{i}@test.com', float(i * 100))
for i in range(batch_size)
]
start_time = time.time()
with self.monitored_transaction(f"PERF_TEST_{description}_{batch_size}") as cursor:
for username, email, balance in test_data:
cursor.execute('''
INSERT INTO users (username, email, balance)
VALUES (?, ?, ?)
''', (username, email, balance))
end_time = time.time()
duration = end_time - start_time
throughput = batch_size / duration
print(f"{description} (批量大小: {batch_size}):")
print(f" 耗时: {duration:.3f}秒")
print(f" 吞吐量: {throughput:.0f} 记录/秒")
# 清理测试数据
cursor = self.connection.cursor()
cursor.execute("DELETE FROM users WHERE username LIKE 'TEST_USER_%'")
self.connection.commit()
# 测试不同批量大小
performance_test(100, "小批量")
performance_test(1000, "中批量")
performance_test(5000, "大批量")
def cleanup(self):
"""清理资源"""
if self.connection:
self.connection.close()
logger.info("🔒 数据库连接已关闭")
@staticmethod
def cleanup_database():
"""清理演示数据库文件"""
import os
db_files = ["best_practices_demo.db", "best_practices_demo.db-wal", "best_practices_demo.db-shm"]
for file in db_files:
if os.path.exists(file):
os.remove(file)
print(f"🗑️ 已删除: {file}")
if __name__ == "__main__":
demo = TransactionBestPractices()
try:
# 短事务演示
demo.short_transaction_demo()
# 原子性事务演示
demo.atomic_transaction_demo()
# 批量操作演示
demo.batch_operation_demo()
# 性能优化演示
demo.performance_optimization_demo()
# 事务指标报告
demo.transaction_metrics_report()
finally:
demo.cleanup()
TransactionBestPractices.cleanup_database()
print("\n🎉 最佳实践演示完成！")
---
2.3 上下文管理器
01.上下文管理器概念
a.基本定义
a.资源管理机制
上下文管理器是一种Python语言特性，通过__enter__和__exit__方法实现资源的自动获取和释放，确保资源在使用后能够正确清理。
b.事务管理应用
在数据库事务管理中，上下文管理器可以自动处理事务的开始、提交和回滚，简化代码并确保事务的正确执行。
c.with语句支持
通过with语句使用上下文管理器，提供了简洁、安全的资源管理方式，即使在发生异常时也能正确清理资源。
b.工作原理
a.__enter__方法
进入上下文时自动调用，通常用于获取资源、开始事务等初始化操作，其返回值会作为as子句的目标。
b.__exit__方法
退出上下文时自动调用，无论是否发生异常都会执行，用于清理资源、提交或回滚事务等收尾操作。
c.异常处理机制
__exit__方法接收异常信息，可以根据异常类型决定是否处理异常、重新抛出异常或执行恢复操作。
c.代码示例
---
# 上下文管理器基础演示
import sqlite3
from contextlib import contextmanager
from typing import Optional
class DatabaseConnection:
def __init__(self, db_path: str):
"""数据库连接类"""
self.db_path = db_path
self.connection: Optional[sqlite3.Connection] = None
def __enter__(self) -> sqlite3.Connection:
"""进入上下文：建立数据库连接"""
print(f"🔗 建立数据库连接: {self.db_path}")
self.connection = sqlite3.connect(self.db_path)
# 配置连接参数
cursor = self.connection.cursor()
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
return self.connection
def __exit__(self, exc_type, exc_val, exc_tb):
"""退出上下文：关闭数据库连接"""
if self.connection:
if exc_type is None:
print("✅ 上下文正常退出，提交所有更改")
self.connection.commit()
else:
print(f"❌ 上下文异常退出: {exc_val}")
print("🔄 回滚所有更改")
self.connection.rollback()
self.connection.close()
self.connection = None
print("🔒 数据库连接已关闭")
# 不处理异常，让调用者处理
return False
@contextmanager
def transaction_scope(connection: sqlite3.Connection):
"""事务作用域上下文管理器"""
cursor = connection.cursor()
try:
print("🚀 开始事务")
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
yield cursor # 提供游标给with块使用
print("✅ 事务提交")
connection.commit()
except Exception as e:
print(f"❌ 事务回滚: {e}")
connection.rollback()
raise
def basic_context_manager_demo():
"""基础上下文管理器演示"""
print("=== 基础上下文管理器演示 ===")
# 使用数据库连接上下文管理器
with DatabaseConnection(':memory:') as conn:
cursor = conn.cursor()
# 创建表
cursor.execute('''
CREATE TABLE employees (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
department TEXT NOT NULL,
salary REAL NOT NULL
)
''')
print("📋 创建员工表")
# 插入数据
employees = [
('张三', '技术部', 8000.0),
('李四', '销售部', 6000.0),
('王五', '技术部', 9000.0)
]
cursor.executemany('''
INSERT INTO employees (name, department, salary)
VALUES (?, ?, ?)
''', employees)
print(f"📝 插入 {len(employees)} 条员工记录")
# 嵌套使用事务上下文管理器
with transaction_scope(conn) as tx_cursor:
# 更新薪资
tx_cursor.execute('''
UPDATE employees SET salary = salary * 1.1
WHERE department = '技术部'
''')
print("💰 技术部薪资上调10%")
# 查询结果
tx_cursor.execute('SELECT name, salary FROM employees ORDER BY salary DESC')
high_earners = tx_cursor.fetchall()
print("高薪员工列表:")
for name, salary in high_earners:
print(f" {name}: ¥{salary:.2f}")
print("🎉 基础演示完成")
def exception_handling_demo():
"""异常处理演示"""
print("\n=== 异常处理演示 ===")
try:
with DatabaseConnection(':memory:') as conn:
cursor = conn.cursor()
# 创建表
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
# 插入初始数据
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Alice', 1000.0))
print("💰 创建账户: Alice - ¥1000")
# 模拟异常操作
with transaction_scope(conn) as tx_cursor:
tx_cursor.execute('UPDATE accounts SET balance = balance - 500 WHERE name = ?', ('Alice',))
print("💸 Alice转出500元")
# 故意违反约束（余额变为负数）
tx_cursor.execute('UPDATE accounts SET balance = balance - 600 WHERE name = ?', ('Alice',))
print("💸 Alice尝试转出600元（将导致余额为负）")
except Exception as e:
print(f"🔍 捕获异常: {e}")
# 验证数据状态（应该被回滚）
with DatabaseConnection(':memory:') as conn:
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE accounts (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
cursor.execute('INSERT INTO accounts (name, balance) VALUES (?, ?)', ('Alice', 1000.0))
conn.commit()
cursor.execute('SELECT balance FROM accounts WHERE name = ?', ('Alice',))
balance = cursor.fetchone()[0]
print(f"📊 验证结果: Alice余额仍为 ¥{balance}（数据未被破坏）")
if __name__ == "__main__":
basic_context_manager_demo()
exception_handling_demo()
---
02.自定义事务上下文管理器
a.高级事务管理器设计
a.事务装饰器模式
将事务控制逻辑封装在装饰器中，通过注解方式为方法添加事务支持，简化业务代码编写。
b.嵌套事务支持
支持事务的嵌套调用，内部事务失败时能够正确回滚到外部事务的安全状态，保持数据一致性。
c.事务监控机制
集成事务执行时间、操作数量、成功率等监控指标，为系统性能优化和问题诊断提供数据支持。
b.配置参数化
a.隔离级别控制
允许动态设置事务的隔离级别，根据业务需求在性能和数据一致性之间找到平衡。
b.超时时间设置
为事务设置合理的超时时间，防止长时间运行的事务占用系统资源，影响整体系统性能。
c.重试策略配置
针对可恢复的错误（如死锁、超时等）实现自动重试机制，提高系统的可靠性和可用性。
c.代码示例
---
# 高级事务上下文管理器
import sqlite3
import time
import logging
from contextlib import contextmanager
from typing import Optional, Dict, Any, Callable
from functools import wraps
from enum import Enum
# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)
class IsolationLevel(Enum):
"""事务隔离级别枚举"""
DEFERRED = "DEFERRED"
IMMEDIATE = "IMMEDIATE"
EXCLUSIVE = "EXCLUSIVE"
class TransactionConfig:
"""事务配置类"""
def __init__(self,
isolation_level: IsolationLevel = IsolationLevel.DEFERRED,
timeout: float = 30.0,
retry_count: int = 3,
retry_delay: float = 0.1,
auto_commit: bool = False,
enable_monitoring: bool = True):
self.isolation_level = isolation_level
self.timeout = timeout
self.retry_count = retry_count
self.retry_delay = retry_delay
self.auto_commit = auto_commit
self.enable_monitoring = enable_monitoring
class TransactionMetrics:
"""事务指标类"""
def __init__(self, transaction_id: str):
self.transaction_id = transaction_id
self.start_time = time.time()
self.end_time: Optional[float] = None
self.operations_count = 0
self.status = "RUNNING"
self.error_message: Optional[str] = None
self.retry_count = 0
def complete(self, success: bool, error_message: Optional[str] = None):
"""完成事务"""
self.end_time = time.time()
self.status = "SUCCESS" if success else "FAILED"
self.error_message = error_message
@property
def duration(self) -> float:
"""获取事务持续时间"""
end = self.end_time or time.time()
return end - self.start_time
class AdvancedTransactionManager:
"""高级事务管理器"""
def __init__(self, db_path: str, default_config: Optional[TransactionConfig] = None):
"""初始化事务管理器"""
self.db_path = db_path
self.default_config = default_config or TransactionConfig()
self.active_transactions: Dict[str, TransactionMetrics] = {}
self.connection_pool = {}
# 预热连接池
self._warmup_connection_pool()
def _warmup_connection_pool(self):
"""预热连接池"""
try:
conn = sqlite3.connect(self.db_path, timeout=self.default_config.timeout)
conn.execute('PRAGMA foreign_keys = ON')
conn.execute('PRAGMA journal_mode = WAL')
conn.close()
logger.info("🔥 连接池预热完成")
except Exception as e:
logger.error(f"❌ 连接池预热失败: {e}")
def get_connection(self) -> sqlite3.Connection:
"""获取数据库连接"""
thread_id = str(id(threading.current_thread()))
if thread_id not in self.connection_pool:
conn = sqlite3.connect(
self.db_path,
timeout=self.default_config.timeout,
check_same_thread=False
)
conn.execute('PRAGMA foreign_keys = ON')
conn.execute('PRAGMA journal_mode = WAL')
conn.execute('PRAGMA synchronous = NORMAL')
self.connection_pool[thread_id] = conn
logger.info(f"🔗 为线程 {thread_id} 创建新连接")
return self.connection_pool[thread_id]
@contextmanager
def transaction(self, config: Optional[TransactionConfig] = None):
"""事务上下文管理器"""
if config is None:
config = self.default_config
transaction_id = f"TX_{int(time.time() * 1000000)}"
metrics = TransactionMetrics(transaction_id)
self.active_transactions[transaction_id] = metrics
conn = self.get_connection()
cursor = conn.cursor()
try:
logger.info(f"🚀 开始事务: {transaction_id}")
# 设置隔离级别
if config.isolation_level == IsolationLevel.IMMEDIATE:
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
elif config.isolation_level == IsolationLevel.EXCLUSIVE:
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
else:
cursor.execute('BEGIN DEFERRED TRANSACTION')
yield TransactionContext(cursor, metrics, config)
# 提交事务
if not config.auto_commit:
conn.commit()
metrics.complete(True)
logger.info(f"✅ 事务提交成功: {transaction_id} (耗时: {metrics.duration:.3f}秒)")
except Exception as e:
# 回滚事务
if not config.auto_commit:
conn.rollback()
metrics.complete(False, str(e))
logger.error(f"❌ 事务回滚: {transaction_id} - {e}")
# 判断是否需要重试
if self._should_retry(e, config):
metrics.retry_count += 1
if metrics.retry_count <= config.retry_count:
logger.info(f"🔄 重试事务: {transaction_id} (第 {metrics.retry_count} 次)")
time.sleep(config.retry_delay * metrics.retry_count)
# 重新进入事务上下文
with self.transaction(config) as retry_context:
retry_context.cursor.execute("SELECT 1") # 空操作保持上下文
else:
logger.error(f"❌ 事务重试次数超限: {transaction_id}")
raise
finally:
# 清理事务记录
if transaction_id in self.active_transactions:
del self.active_transactions[transaction_id]
def _should_retry(self, exception: Exception, config: TransactionConfig) -> bool:
"""判断是否应该重试"""
retry_errors = [
"database is locked",
"SQLITE_BUSY",
"SQLITE_LOCKED",
"connection timed out"
]
error_msg = str(exception).lower()
return any(error in error_msg for error in retry_errors) and config.retry_count > 0
def transaction_decorator(self, config: Optional[TransactionConfig] = None):
"""事务装饰器"""
def decorator(func: Callable):
@wraps(func)
def wrapper(*args, **kwargs):
with self.transaction(config) as tx_context:
# 将事务上下文作为第一个参数传递给函数
return func(tx_context, *args, **kwargs)
return wrapper
return decorator
def get_transaction_statistics(self) -> Dict[str, Any]:
"""获取事务统计信息"""
if not self.active_transactions:
return {}
return {
'active_count': len(self.active_transactions),
'active_transactions': [
{
'id': tx.transaction_id,
'duration': tx.duration,
'operations': tx.operations_count,
'status': tx.status
}
for tx in self.active_transactions.values()
]
}
class TransactionContext:
"""事务上下文类"""
def __init__(self, cursor: sqlite3.Cursor, metrics: TransactionMetrics, config: TransactionConfig):
self.cursor = cursor
self.metrics = metrics
self.config = config
def execute(self, sql: str, parameters=()) -> sqlite3.Cursor:
"""执行SQL并记录指标"""
self.metrics.operations_count += 1
if self.config.enable_monitoring:
logger.debug(f"📝 执行SQL: {sql}")
return self.cursor.execute(sql, parameters)
def executemany(self, sql: str, parameters: list) -> sqlite3.Cursor:
"""批量执行SQL并记录指标"""
self.metrics.operations_count += len(parameters)
if self.config.enable_monitoring:
logger.debug(f"📝 批量执行SQL: {sql} (参数数量: {len(parameters)})")
return self.cursor.executemany(sql, parameters)
def commit(self):
"""手动提交"""
if not self.config.auto_commit:
self.cursor.connection.commit()
logger.info("📤 手动提交事务")
def rollback(self):
"""手动回滚"""
if not self.config.auto_commit:
self.cursor.connection.rollback()
logger.info("🔄 手动回滚事务")
# 使用示例
def advanced_transaction_demo():
"""高级事务管理器演示"""
print("=== 高级事务管理器演示 ===")
# 创建事务管理器
config = TransactionConfig(
isolation_level=IsolationLevel.IMMEDIATE,
timeout=10.0,
retry_count=3,
enable_monitoring=True
)
manager = AdvancedTransactionManager(":memory:", config)
try:
# 创建表结构
with manager.transaction() as tx:
tx.execute('''
CREATE TABLE products (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
price REAL NOT NULL CHECK(price > 0),
stock INTEGER NOT NULL CHECK(stock >= 0),
category TEXT NOT NULL
)
''')
tx.execute('''
CREATE TABLE orders (
id INTEGER PRIMARY KEY,
product_id INTEGER,
quantity INTEGER NOT NULL CHECK(quantity > 0),
total_price REAL NOT NULL CHECK(total_price > 0),
status TEXT NOT NULL DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
print("📋 创建产品表和订单表")
# 批量插入产品数据
products_data = [
('智能手机', 2999.99, 100, '电子产品'),
('笔记本电脑', 6999.99, 50, '电子产品'),
('无线耳机', 299.99, 200, '电子产品'),
('充电宝', 99.99, 500, '配件')
]
with manager.transaction() as tx:
tx.executemany('''
INSERT INTO products (name, price, stock, category)
VALUES (?, ?, ?, ?)
''', products_data)
print(f"📦 批量插入 {len(products_data)} 个产品")
# 使用装饰器创建订单
@manager.transaction_decorator()
def create_order(tx_context: TransactionContext, product_id: int, quantity: int):
"""创建订单（带事务装饰器）"""
# 检查产品库存
result = tx_context.execute('SELECT name, price, stock FROM products WHERE id = ?', (product_id,))
product = result.fetchone()
if not product:
raise ValueError(f"产品不存在: ID {product_id}")
name, price, stock = product
if stock < quantity:
raise ValueError(f"库存不足: {name}, 当前库存 {stock}, 需求 {quantity}")
# 计算总价
total_price = price * quantity
# 创建订单
tx_context.execute('''
INSERT INTO orders (product_id, quantity, total_price, status)
VALUES (?, ?, ?, ?)
''', (product_id, quantity, total_price, 'confirmed'))
# 更新库存
tx_context.execute('''
UPDATE products SET stock = stock - ? WHERE id = ?
''', (quantity, product_id))
return tx_context.cursor.lastrowid
# 创建多个订单
print("\n🛒 创建订单:")
# 成功订单
try:
order1_id = create_order(1, 2) # 购买2部智能手机
print(f"✅ 订单1创建成功: ID {order1_id}")
except Exception as e:
print(f"❌ 订单1创建失败: {e}")
try:
order2_id = create_order(2, 1) # 购买1台笔记本电脑
print(f"✅ 订单2创建成功: ID {order2_id}")
except Exception as e:
print(f"❌ 订单2创建失败: {e}")
# 库存不足的订单
try:
order3_id = create_order(1, 200) # 尝试购买200部智能手机（库存只有98）
print(f"✅ 订单3创建成功: ID {order3_id}")
except Exception as e:
print(f"❌ 订单3创建失败: {e}")
# 查询最终状态
with manager.transaction() as tx:
result = tx.execute('SELECT name, stock FROM products ORDER BY id')
products = result.fetchall()
print("\n📊 产品库存状态:")
for name, stock in products:
print(f" {name}: {stock}")
result = tx.execute('SELECT COUNT(*) FROM orders WHERE status = "confirmed"')
order_count = result.fetchone()[0]
print(f"\n📋 成功订单数: {order_count}")
# 显示事务统计
stats = manager.get_transaction_statistics()
if stats:
print(f"\n📈 活跃事务: {stats['active_count']}")
if stats['active_transactions']:
for tx_info in stats['active_transactions']:
print(f" {tx_info['id']}: {tx_info['status']}, "
f"耗时 {tx_info['duration']:.3f}s, "
f"操作数 {tx_info['operations']}")
finally:
# 清理资源
for conn in manager.connection_pool.values():
conn.close()
logger.info("🔒 所有连接已关闭")
if __name__ == "__main__":
import threading # 需要导入用于连接管理
advanced_transaction_demo()
---
03.上下文管理器的最佳实践
a.资源管理最佳实践
a.连接池集成
将数据库连接池与上下文管理器结合使用，实现连接的复用和自动管理，提高系统性能。
b.异常处理策略
设计合理的异常处理层次结构，区分可恢复异常和不可恢复异常，实现精确的错误处理和恢复策略。
c.资源清理保证
确保在任何情况下都能正确清理资源，包括数据库连接、临时文件、内存对象等，防止资源泄漏。
b.性能优化技巧
a.延迟初始化
对于资源密集型的初始化操作，采用延迟初始化策略，只在真正需要时才创建资源，提高启动性能。
b.缓存机制
在上下文管理器中集成适当的缓存机制，减少重复的数据库操作，提高整体执行效率。
c.批量操作优化
支持批量操作的上下文管理，将多个相关操作组合在一起执行，减少事务开销。
c.代码示例
---
# 上下文管理器最佳实践演示
import sqlite3
import time
import threading
import queue
from contextlib import contextmanager
from typing import Optional, Dict, Any, List, Union
from dataclasses import dataclass, field
from enum import Enum, auto
import weakref
import gc
class ResourceState(Enum):
"""资源状态枚举"""
INITIALIZING = auto()
READY = auto()
BUSY = auto()
ERROR = auto()
DISPOSED = auto()
@dataclass
class ResourceMetrics:
"""资源使用指标"""
created_at: float = field(default_factory=time.time)
last_used: float = field(default_factory=time.time)
usage_count: int = 0
total_operations: int = 0
errors: int = 0
avg_operation_time: float = 0.0
class DatabaseConnectionPool:
"""数据库连接池（最佳实践实现）"""
def __init__(self, db_path: str, max_connections: int = 10, timeout: float = 30.0):
self.db_path = db_path
self.max_connections = max_connections
self.timeout = timeout
self._pool = queue.Queue(maxsize=max_connections)
self._all_connections = weakref.WeakSet()
self._lock = threading.RLock()
self._metrics = ResourceMetrics()
# 预创建连接
self._initialize_pool()
def _initialize_pool(self):
"""初始化连接池"""
for i in range(min(3, self.max_connections)): # 预创建3个连接
self._create_connection()
def _create_connection(self) -> sqlite3.Connection:
"""创建新的数据库连接"""
try:
conn = sqlite3.connect(
self.db_path,
timeout=self.timeout,
check_same_thread=False,
isolation_level="DEFERRED"
)
# 优化配置
cursor = conn.cursor()
cursor.execute('PRAGMA foreign_keys = ON')
cursor.execute('PRAGMA journal_mode = WAL')
cursor.execute('PRAGMA synchronous = NORMAL')
cursor.execute('PRAGMA cache_size = 10000')
cursor.execute('PRAGMA temp_store = MEMORY')
# 设置row factory便于数据访问
conn.row_factory = sqlite3.Row
self._all_connections.add(conn)
return conn
except Exception as e:
raise RuntimeError(f"创建数据库连接失败: {e}")
@contextmanager
def get_connection(self):
"""获取连接的上下文管理器"""
conn = None
try:
# 从池中获取连接
conn = self._pool.get(timeout=self.timeout)
self._metrics.usage_count += 1
self._metrics.last_used = time.time()
# 验证连接是否仍然有效
try:
conn.execute("SELECT 1")
except sqlite3.Error:
# 连接已失效，创建新连接
conn.close()
conn = self._create_connection()
yield conn
except queue.Empty:
raise RuntimeError(f"获取数据库连接超时 (等待时间: {self.timeout}秒)")
except Exception as e:
self._metrics.errors += 1
raise
finally:
if conn:
# 将连接返回池中
try:
self._pool.put(conn, timeout=1.0)
except queue.Full:
# 池已满，直接关闭连接
conn.close()
def get_metrics(self) -> Dict[str, Any]:
"""获取连接池指标"""
with self._lock:
return {
'pool_size': self._pool.qsize(),
'total_connections': len(self._all_connections),
'usage_count': self._metrics.usage_count,
'total_operations': self._metrics.total_operations,
'errors': self._metrics.errors,
'uptime': time.time() - self._metrics.created_at
}
def close_all(self):
"""关闭所有连接"""
with self._lock:
while not self._pool.empty():
try:
conn = self._pool.get_nowait()
conn.close()
except queue.Empty:
break
self._all_connections.clear()
class OptimizedTransactionManager:
"""优化的事务管理器"""
def __init__(self, connection_pool: DatabaseConnectionPool):
self.connection_pool = connection_pool
self._transaction_cache = {}
self._batch_operations = {}
self._lock = threading.RLock()
@contextmanager
def transaction(self, isolation_level: str = "DEFERRED", batch_size: Optional[int] = None):
"""优化的事务上下文管理器"""
with self.connection_pool.get_connection() as conn:
cursor = conn.cursor()
try:
# 设置隔离级别
if isolation_level == "IMMEDIATE":
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
elif isolation_level == "EXCLUSIVE":
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
else:
cursor.execute('BEGIN DEFERRED TRANSACTION')
yield OptimizedTransactionContext(cursor, self, batch_size)
# 检查是否有批量操作需要提交
transaction_id = id(cursor)
if transaction_id in self._batch_operations:
self._execute_batch_operations(cursor, transaction_id)
del self._batch_operations[transaction_id]
conn.commit()
except Exception as e:
conn.rollback()
# 清理批量操作
transaction_id = id(cursor)
if transaction_id in self._batch_operations:
del self._batch_operations[transaction_id]
raise
def _execute_batch_operations(self, cursor: sqlite3.Connection.cursor, transaction_id: int):
"""执行批量操作"""
batch_ops = self._batch_operations.get(transaction_id, {})
for operation_type, operations in batch_ops.items():
if not operations:
continue
start_time = time.time()
if operation_type == "INSERT":
cursor.executemany(operations['sql'], operations['data'])
elif operation_type == "UPDATE":
for sql, params in operations:
cursor.execute(sql, params)
elif operation_type == "DELETE":
cursor.executemany(operations['sql'], operations['data'])
operation_time = time.time() - start_time
print(f"📊 批量{operation_type}操作: {len(operations.get('data', operations))}条记录, 耗时: {operation_time:.3f}秒")
class OptimizedTransactionContext:
"""优化的事务上下文"""
def __init__(self, cursor: sqlite3.Cursor, manager: OptimizedTransactionManager, batch_size: Optional[int] = None):
self.cursor = cursor
self.manager = manager
self.batch_size = batch_size
self.transaction_id = id(cursor)
self._operation_cache = {}
def execute(self, sql: str, parameters=()) -> sqlite3.Cursor:
"""执行SQL（带缓存）"""
# 简单的查询缓存
cache_key = (sql.strip().upper(), tuple(parameters) if parameters else ())
if sql.strip().upper().startswith('SELECT') and cache_key in self._operation_cache:
return self._operation_cache[cache_key]
result = self.cursor.execute(sql, parameters)
if sql.strip().upper().startswith('SELECT'):
self._operation_cache[cache_key] = result
return result
def batch_insert(self, table: str, data: List[Dict[str, Any]]):
"""批量插入"""
if not data:
return
# 动态构建INSERT语句
columns = list(data[0].keys())
placeholders = ', '.join(['?'] * len(columns))
sql = f'INSERT INTO {table} ({", ".join(columns)}) VALUES ({placeholders})'
if self.batch_size and len(data) > self.batch_size:
# 大批量数据，使用批量操作
self._add_batch_operation("INSERT", sql, data)
else:
# 小批量数据，直接执行
self.cursor.executemany(sql, [tuple(row[col] for col in columns) for row in data])
def batch_update(self, table: str, updates: List[Dict[str, Any]], condition_column: str):
"""批量更新"""
if not updates:
return
for update in updates:
condition_value = update.get(condition_column)
if condition_value is None:
continue
# 构建SET子句
set_clauses = []
values = []
for column, value in update.items():
if column != condition_column:
set_clauses.append(f"{column} = ?")
values.append(value)
if set_clauses:
sql = f'UPDATE {table} SET {", ".join(set_clauses)} WHERE {condition_column} = ?'
values.append(condition_value)
if self.batch_size and len(updates) > self.batch_size:
self._add_batch_operation("UPDATE", sql, values)
else:
self.cursor.execute(sql, values)
def _add_batch_operation(self, operation_type: str, sql: str, data: Any):
"""添加批量操作到队列"""
with self.manager._lock:
if self.transaction_id not in self.manager._batch_operations:
self.manager._batch_operations[self.transaction_id] = {}
if operation_type not in self.manager._batch_operations[self.transaction_id]:
self.manager._batch_operations[self.transaction_id][operation_type] = {
'sql': sql,
'data': []
}
self.manager._batch_operations[self.transaction_id][operation_type]['data'].append(data)
def best_practices_demo():
"""最佳实践演示"""
print("=== 上下文管理器最佳实践演示 ===")
# 创建连接池
pool = DatabaseConnectionPool(':memory:', max_connections=5, timeout=10.0)
manager = OptimizedTransactionManager(pool)
try:
# 初始化数据库结构
with manager.transaction() as tx:
tx.execute('''
CREATE TABLE IF NOT EXISTS customers (
id INTEGER PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
email TEXT UNIQUE NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
''')
tx.execute('''
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY AUTOINCREMENT,
customer_id INTEGER NOT NULL,
product_name TEXT NOT NULL,
quantity INTEGER NOT NULL,
price REAL NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (customer_id) REFERENCES customers(id)
)
''')
print("📋 创建数据库结构")
# 批量插入客户数据（演示批量操作优化）
print("\n👥 批量插入客户数据:")
customers_data = [
{'name': f'客户{i}', 'email': f'customer{i}@example.com'}
for i in range(1, 1001)
]
start_time = time.time()
with manager.transaction(batch_size=100) as tx:
tx.batch_insert('customers', customers_data)
batch_time = time.time() - start_time
print(f"✅ 批量插入1000个客户，耗时: {batch_time:.3f}秒")
# 创建订单数据
print("\n🛒 批量创建订单:")
orders_data = []
for i in range(1, 501):
orders_data.append({
'customer_id': (i % 1000) + 1, # 随机客户ID
'product_name': f'产品{i % 10}',
'quantity': (i % 5) + 1,
'price': float((i % 100) + 1) * 10
})
start_time = time.time()
with manager.transaction(batch_size=50) as tx:
tx.batch_insert('orders', orders_data)
orders_time = time.time() - start_time
print(f"✅ 批量创建500个订单，耗时: {orders_time:.3f}秒")
# 批量更新订单状态
print("\n📊 批量更新订单状态:")
updates = []
for i in range(1, 501):
status = 'confirmed' if i % 2 == 0 else 'shipped'
updates.append({'status': status, 'id': i})
start_time = time.time()
with manager.transaction() as tx:
tx.batch_update('orders', updates, 'id')
update_time = time.time() - start_time
print(f"✅ 批量更新500个订单状态，耗时: {update_time:.3f}秒")
# 使用缓存查询验证结果
print("\n📈 使用缓存查询验证结果:")
start_time = time.time()
with manager.transaction() as tx:
# 第一次查询（会缓存）
result1 = tx.execute('SELECT COUNT(*) FROM customers WHERE id <= 100')
count1 = result1.fetchone()[0]
# 第二次查询（从缓存获取）
result2 = tx.execute('SELECT COUNT(*) FROM customers WHERE id <= 100')
count2 = result2.fetchone()[0]
cache_time = time.time() - start_time
print(f"📋 缓存查询结果: 客户数量 = {count1}, 耗时: {cache_time:.6f}秒")
# 连接池指标
print("\n📊 连接池指标:")
metrics = pool.get_metrics()
for key, value in metrics.items():
if isinstance(value, float):
print(f" {key}: {value:.3f}")
else:
print(f" {key}: {value}")
# 内存使用情况
print("\n💾 内存使用情况:")
import sys
memory_usage = sys.getsizeof(pool) + sys.getsizeof(manager)
print(f" 管理器对象大小: {memory_usage} 字节")
# 强制垃圾回收
collected = gc.collect()
print(f" 垃圾回收对象数: {collected}")
finally:
# 清理资源
pool.close_all()
print("\n🔒 连接池已关闭")
if __name__ == "__main__":
best_practices_demo()
---
2.4 隔离级别
01.隔离级别概述
a.隔离级别定义
a.概念阐述
隔离级别是数据库事务并发控制的重要机制，它定义了事务之间的可见性规则，即一个事务的修改何时对其他事务可见。不同的隔离级别在并发性能和数据一致性之间提供了不同的平衡。
b.隔离级别分类
SQL标准定义了四个隔离级别：读未提交(Read Uncommitted)、读已提交(Read Committed)、可重复读(Repeatable Read)和串行化(Serializable)。隔离级别从低到高，并发性能逐渐降低，但数据一致性保证逐渐增强。
c.代码示例
---
# 隔离级别概念演示
import sqlite3
import threading
import time
from contextlib import contextmanager
class IsolationLevelDemo:
"""隔离级别演示类"""
def __init__(self, db_name='isolation_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""初始化测试数据库"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 创建测试表
cursor.execute('''
CREATE TABLE IF NOT EXISTS products (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
stock INTEGER NOT NULL,
price REAL NOT NULL
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS orders (
id INTEGER PRIMARY KEY,
product_id INTEGER,
quantity INTEGER NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP,
FOREIGN KEY (product_id) REFERENCES products(id)
)
''')
# 初始化测试数据
cursor.execute('DELETE FROM products')
cursor.execute('DELETE FROM orders')
cursor.executemany(
'INSERT INTO products (id, name, stock, price) VALUES (?, ?, ?, ?)',
[(1, '笔记本电脑', 100, 5999.0), (2, '无线鼠标', 200, 199.0), (3, '机械键盘', 150, 599.0)]
)
conn.commit()
conn.close()
print("测试数据库初始化完成")
@contextmanager
def get_connection(self, isolation_level=None):
"""获取数据库连接的上下文管理器"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
if isolation_level:
# 设置隔离级别
cursor.execute(f'PRAGMA read_uncommitted = {isolation_level}')
try:
yield conn, cursor
finally:
conn.close()
def demonstrate_isolation_concept(self):
"""演示隔离级别的基本概念"""
print("=== 隔离级别概念演示 ===")
def read_transaction():
"""读取事务"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION')
# 读取产品库存
cursor.execute('SELECT name, stock FROM products WHERE id = 1')
product = cursor.fetchone()
print(f"[读取事务] 读取到产品: {product[0]}, 库存: {product[1]}")
# 模拟长时间处理
time.sleep(2)
# 再次读取相同数据
cursor.execute('SELECT name, stock FROM products WHERE id = 1')
product2 = cursor.fetchone()
print(f"[读取事务] 再次读取到产品: {product2[0]}, 库存: {product2[1]}")
conn.commit()
print("[读取事务] 事务完成")
except Exception as e:
print(f"[读取事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def write_transaction():
"""写入事务"""
time.sleep(1) # 延迟启动
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 修改库存
cursor.execute('UPDATE products SET stock = stock - 10 WHERE id = 1')
print("[写入事务] 已修改产品1的库存")
conn.commit()
print("[写入事务] 事务已提交")
except Exception as e:
print(f"[写入事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 启动并发事务
t1 = threading.Thread(target=read_transaction)
t2 = threading.Thread(target=write_transaction)
t1.start()
t2.start()
t1.join()
t2.join()
# 检查最终状态
with self.get_connection() as (conn, cursor):
cursor.execute('SELECT name, stock FROM products WHERE id = 1')
final_product = cursor.fetchone()
print(f"最终产品状态: {final_product[0]}, 库存: {final_product[1]}")
if __name__ == "__main__":
demo = IsolationLevelDemo()
demo.demonstrate_isolation_concept()
---
02.SQLite的隔离级别实现
a.读未提交(Read Uncommitted)
a.特性说明
读未提交是最低的隔离级别，允许一个事务读取另一个事务尚未提交的修改。这种级别可能导致脏读(Dirty Read)，即读取到最终可能被回滚的数据。在SQLite中，可以通过PRAGMA read_uncommitted = 1来启用。
b.适用场景
主要适用于对数据一致性要求不高但需要高并发性能的场景，如统计报表生成、数据分析等。在生产环境中使用需要谨慎评估风险。
c.代码示例
---
# 读未提交隔离级别演示
class ReadUncommittedDemo:
"""读未提交隔离级别演示"""
def __init__(self, db_name='read_uncommitted_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS transactions (
id INTEGER PRIMARY KEY,
amount REAL NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
account_id INTEGER NOT NULL
)
''')
cursor.execute('DELETE FROM transactions')
conn.commit()
conn.close()
def dirty_read_demo(self):
"""演示脏读现象"""
print("=== 脏读现象演示 ===")
def write_transaction():
"""写入未提交数据的事务"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 插入一条交易记录
cursor.execute(
'INSERT INTO transactions (amount, status, account_id) VALUES (?, ?, ?)',
(1000.0, 'pending', 1)
)
print("[写入事务] 插入交易记录，金额: 1000.0")
# 模拟处理时间
time.sleep(3)
print("[写入事务] 模拟异常，准备回滚")
conn.rollback() # 回滚事务
print("[写入事务] 事务已回滚")
except Exception as e:
print(f"[写入事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def read_uncommitted_transaction():
"""读未提交事务"""
time.sleep(1) # 延迟启动
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 启用读未提交模式
cursor.execute('PRAGMA read_uncommitted = 1')
cursor.execute('BEGIN TRANSACTION')
# 读取数据（可能读到未提交的数据）
cursor.execute('SELECT COUNT(*), SUM(amount) FROM transactions WHERE account_id = 1')
result = cursor.fetchone()
print(f"[读未提交] 读取到交易数量: {result[0]}, 总金额: {result[1]}")
# 等待写入事务完成
time.sleep(3)
# 再次读取
cursor.execute('SELECT COUNT(*), SUM(amount) FROM transactions WHERE account_id = 1')
result2 = cursor.fetchone()
print(f"[读未提交] 再次读取交易数量: {result2[0]}, 总金额: {result2[1]}")
conn.commit()
print("[读未提交] 读取事务完成")
except Exception as e:
print(f"[读未提交] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 启动并发事务
t1 = threading.Thread(target=write_transaction)
t2 = threading.Thread(target=read_uncommitted_transaction)
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == "__main__":
demo = ReadUncommittedDemo()
demo.dirty_read_demo()
---
b.读已提交(Read Committed) - SQLite默认
a.特性说明
读已提交是SQLite的默认隔离级别。它确保事务只能读取其他事务已提交的修改，避免了脏读问题。但是在同一事务中多次读取相同数据可能得到不同的结果，这被称为不可重复读(Non-repeatable Read)。
b.实现机制
SQLite通过MVCC(Multi-Version Concurrency Control)机制实现读已提交隔离。每次读操作都会看到在事务开始时已提交的数据快照，写操作会获取适当的锁来保证数据一致性。
c.代码示例
---
# 读已提交隔离级别演示
class ReadCommittedDemo:
"""读已提交隔离级别演示"""
def __init__(self, db_name='read_committed_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS inventory (
product_id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
quantity INTEGER NOT NULL,
last_updated DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 初始化数据
cursor.execute('DELETE FROM inventory')
cursor.execute(
'INSERT INTO inventory (product_id, name, quantity) VALUES (?, ?, ?)',
(1, '智能手机', 50)
)
conn.commit()
conn.close()
def non_repeatable_read_demo(self):
"""演示不可重复读现象"""
print("=== 不可重复读现象演示 ===")
def long_read_transaction():
"""长时间运行的读事务"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION')
# 第一次读取
cursor.execute('SELECT name, quantity FROM inventory WHERE product_id = 1')
result1 = cursor.fetchone()
print(f"[长读事务] 第一次读取: {result1[0]}, 数量: {result1[1]}")
# 模拟长时间处理
time.sleep(2)
# 第二次读取（可能得到不同的结果）
cursor.execute('SELECT name, quantity FROM inventory WHERE product_id = 1')
result2 = cursor.fetchone()
print(f"[长读事务] 第二次读取: {result2[0]}, 数量: {result2[1]}")
if result1[1] != result2[1]:
print(f"[长读事务] ❌ 检测到不可重复读: {result1[1]} -> {result2[1]}")
else:
print("[长读事务] ✅ 数据一致，没有不可重复读")
conn.commit()
except Exception as e:
print(f"[长读事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def update_transaction():
"""更新事务"""
time.sleep(1) # 延迟启动
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 更新库存
cursor.execute(
'UPDATE inventory SET quantity = quantity - 5 WHERE product_id = 1'
)
print("[更新事务] 减少库存5个单位")
conn.commit()
print("[更新事务] 更新已提交")
except Exception as e:
print(f"[更新事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 启动并发事务
t1 = threading.Thread(target=long_read_transaction)
t2 = threading.Thread(target=update_transaction)
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == "__main__":
demo = ReadCommittedDemo()
demo.non_repeatable_read_demo()
---
c.可重复读(Repeatable Read)
a.特性说明
可重复读隔离级别确保在同一事务中多次读取相同数据会得到一致的结果，避免了不可重复读问题。但是可能出现幻读(Phantom Read)，即在事务执行期间，其他事务插入的新记录可能出现在后续查询中。
b.SQLite的实现
SQLite通过在事务开始时创建数据快照来实现可重复读。这种机制确保了事务期间看到的数据视图保持一致，但不阻止其他事务插入新记录。
c.代码示例
---
# 可重复读隔离级别演示
class RepeatableReadDemo:
"""可重复读隔离级别演示"""
def __init__(self, db_name='repeatable_read_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS employees (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
department TEXT NOT NULL,
salary REAL NOT NULL
)
''')
# 初始化数据
cursor.execute('DELETE FROM employees')
cursor.executemany(
'INSERT INTO employees (name, department, salary) VALUES (?, ?, ?)',
[('张三', '技术部', 8000), ('李四', '技术部', 7500), ('王五', '销售部', 6000)]
)
conn.commit()
conn.close()
def phantom_read_demo(self):
"""演示幻读现象"""
print("=== 幻读现象演示 ===")
def consistent_read_transaction():
"""一致性读取事务"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION')
# 第一次查询技术部员工
cursor.execute(
'SELECT name, salary FROM employees WHERE department = ? ORDER BY name',
('技术部',)
)
tech_employees1 = cursor.fetchall()
print(f"[一致读事务] 第一次查询技术部员工: {len(tech_employees1)}人")
for emp in tech_employees1:
print(f" {emp[0]}: {emp[1]}")
# 模拟长时间处理
time.sleep(2)
# 第二次查询（可能出现幻读）
cursor.execute(
'SELECT name, salary FROM employees WHERE department = ? ORDER BY name',
('技术部',)
)
tech_employees2 = cursor.fetchall()
print(f"[一致读事务] 第二次查询技术部员工: {len(tech_employees2)}人")
for emp in tech_employees2:
print(f" {emp[0]}: {emp[1]}")
if len(tech_employees1) != len(tech_employees2):
print(f"[一致读事务] ❌ 检测到幻读: {len(tech_employees1)} -> {len(tech_employees2)}人")
else:
print("[一致读事务] ✅ 数据一致，没有幻读")
conn.commit()
except Exception as e:
print(f"[一致读事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def insert_transaction():
"""插入新员工事务"""
time.sleep(1) # 延迟启动
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 插入新的技术部员工
cursor.execute(
'INSERT INTO employees (name, department, salary) VALUES (?, ?, ?)',
('赵六', '技术部', 8500)
)
print("[插入事务] 插入新员工: 赵六, 技术部, 8500")
conn.commit()
print("[插入事务] 插入已提交")
except Exception as e:
print(f"[插入事务] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 启动并发事务
t1 = threading.Thread(target=consistent_read_transaction)
t2 = threading.Thread(target=insert_transaction)
t1.start()
t2.start()
t1.join()
t2.join()
if __name__ == "__main__":
demo = RepeatableReadDemo()
demo.phantom_read_demo()
---
d.串行化(Serializable)
a.特性说明
串行化是最高的隔离级别，它确保事务的执行结果与串行执行完全相同。通过严格的锁定机制避免了所有的并发问题，包括脏读、不可重复读和幻读。
b.性能影响
这种隔离级别会显著降低并发性能，因为需要对更多的资源进行加锁。在SQLite中，串行化通常通过BEGIN IMMEDIATE或BEGIN EXCLUSIVE来实现。
c.代码示例
---
# 串行化隔离级别演示
class SerializableDemo:
"""串行化隔离级别演示"""
def __init__(self, db_name='serializable_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS bank_accounts (
account_id INTEGER PRIMARY KEY,
account_number TEXT NOT NULL UNIQUE,
balance REAL NOT NULL CHECK(balance >= 0)
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS transaction_log (
id INTEGER PRIMARY KEY,
from_account TEXT NOT NULL,
to_account TEXT NOT NULL,
amount REAL NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 初始化数据
cursor.execute('DELETE FROM bank_accounts')
cursor.execute('DELETE FROM transaction_log')
cursor.executemany(
'INSERT INTO bank_accounts (account_number, balance) VALUES (?, ?)',
[('ACC001', 10000.0), ('ACC002', 5000.0), ('ACC003', 8000.0)]
)
conn.commit()
conn.close()
def serializable_transfer_demo(self):
"""演示串行化转账操作"""
print("=== 串行化转账操作演示 ===")
def transfer_funds(from_acc, to_acc, amount, delay=0):
"""串行化转账函数"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 使用排他锁确保串行化
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
print(f"[转账{from_acc}→{to_acc}] 获取排他锁")
# 检查源账户余额
cursor.execute(
'SELECT balance FROM bank_accounts WHERE account_number = ?',
(from_acc,)
)
from_balance = cursor.fetchone()[0]
print(f"[转账{from_acc}→{to_acc}] 源账户余额: {from_balance}")
if from_balance < amount:
print(f"[转账{from_acc}→{to_acc}] ❌ 余额不足")
conn.rollback()
return False
# 模拟处理时间
time.sleep(delay)
# 执行转账
cursor.execute(
'UPDATE bank_accounts SET balance = balance - ? WHERE account_number = ?',
(amount, from_acc)
)
cursor.execute(
'UPDATE bank_accounts SET balance = balance + ? WHERE account_number = ?',
(amount, to_acc)
)
# 记录转账日志
cursor.execute(
'INSERT INTO transaction_log (from_account, to_account, amount) VALUES (?, ?, ?)',
(from_acc, to_acc, amount)
)
conn.commit()
print(f"[转账{from_acc}→{to_acc}] ✅ 转账成功: {amount}")
return True
except Exception as e:
print(f"[转账{from_acc}→{to_acc}] 异常: {e}")
if 'conn' in locals():
conn.rollback()
return False
finally:
if 'conn' in locals():
conn.close()
def check_balances():
"""检查所有账户余额"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('SELECT account_number, balance FROM bank_accounts ORDER BY account_number')
accounts = cursor.fetchall()
total_balance = sum(acc[1] for acc in accounts)
print(f"[余额检查] 总余额: {total_balance}")
for acc in accounts:
print(f" {acc[0]}: {acc[1]}")
conn.close()
return total_balance
# 检查初始状态
print("初始余额状态:")
initial_total = check_balances()
# 并发执行多个转账操作
transfers = [
('ACC001', 'ACC002', 1000.0, 1),
('ACC002', 'ACC003', 500.0, 0.5),
('ACC003', 'ACC001', 200.0, 0.8)
]
threads = []
for from_acc, to_acc, amount, delay in transfers:
thread = threading.Thread(
target=transfer_funds,
args=(from_acc, to_acc, amount, delay)
)
threads.append(thread)
print("\n开始并发转账操作...")
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# 检查最终状态
print("\n最终余额状态:")
final_total = check_balances()
# 验证余额一致性
if abs(initial_total - final_total) < 0.01:
print("✅ 余额一致性验证通过")
else:
print("❌ 余额一致性验证失败")
# 显示转账日志
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('SELECT from_account, to_account, amount, timestamp FROM transaction_log ORDER BY timestamp')
logs = cursor.fetchall()
print("\n转账日志:")
for log in logs:
print(f" {log[0]} → {log[1]}: {log[2]} ({log[3]})")
conn.close()
if __name__ == "__main__":
demo = SerializableDemo()
demo.serializable_transfer_demo()
---
03.隔离级别选择策略
a.业务场景分析
a.读多写少场景
对于报表生成、数据分析等以读操作为主的场景，可以使用读已提交隔离级别，在保证数据基本一致性的同时提供较好的并发性能。
b.高并发更新场景
对于银行交易、库存管理等高并发更新场景，建议使用串行化或至少可重复读隔离级别，确保数据的强一致性。
c.代码示例
---
# 隔离级别选择策略演示
class IsolationLevelSelector:
"""隔离级别选择器"""
def __init__(self, db_name='isolation_selector_demo.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS products (
id INTEGER PRIMARY KEY,
name TEXT NOT NULL,
category TEXT NOT NULL,
stock INTEGER NOT NULL,
price REAL NOT NULL,
version INTEGER NOT NULL DEFAULT 1
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS sales (
id INTEGER PRIMARY KEY,
product_id INTEGER NOT NULL,
quantity INTEGER NOT NULL,
unit_price REAL NOT NULL,
total_price REAL NOT NULL,
sale_time DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
# 初始化商品数据
cursor.execute('DELETE FROM products')
cursor.execute('DELETE FROM sales')
products_data = [
(1, '智能手机', '电子产品', 100, 2999.0),
(2, '笔记本电脑', '电子产品', 50, 5999.0),
(3, '运动鞋', '服装', 200, 399.0),
(4, '图书', '文具', 500, 59.0)
]
cursor.executemany(
'INSERT INTO products (id, name, category, stock, price) VALUES (?, ?, ?, ?, ?)',
products_data
)
conn.commit()
conn.close()
def report_generation_scenario(self):
"""报表生成场景：使用读已提交"""
print("=== 报表生成场景（读已提交）===")
def generate_report():
"""生成销售报表"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION') # 使用默认的读已提交
# 按类别统计商品库存
cursor.execute('''
SELECT category, COUNT(*) as product_count, SUM(stock) as total_stock, AVG(price) as avg_price
FROM products
GROUP BY category
ORDER BY total_stock DESC
''')
category_stats = cursor.fetchall()
print("[报表生成] 商品分类统计:")
for stat in category_stats:
print(f" {stat[0]}: {stat[1]}种商品, 总库存: {stat[2]}, 平均价格: {stat[3]:.2f}")
# 计算总库存价值
cursor.execute('SELECT SUM(stock * price) FROM products')
total_value = cursor.fetchone()[0]
print(f"[报表生成] 总库存价值: {total_value:.2f}")
conn.commit()
print("[报表生成] 报表生成完成")
except Exception as e:
print(f"[报表生成] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def concurrent_sale():
"""并发销售操作"""
time.sleep(0.5) # 延迟启动
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
# 销售一部智能手机
cursor.execute('''
INSERT INTO sales (product_id, quantity, unit_price, total_price)
VALUES (?, 1, ?, ?)
''', (1, 2999.0, 2999.0))
cursor.execute('UPDATE products SET stock = stock - 1 WHERE id = 1')
print("[并发销售] 销售了1部智能手机")
conn.commit()
print("[并发销售] 销售记录已提交")
except Exception as e:
print(f"[并发销售] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 启动报表生成和并发销售
t1 = threading.Thread(target=generate_report)
t2 = threading.Thread(target=concurrent_sale)
t1.start()
t2.start()
t1.join()
t2.join()
def inventory_management_scenario(self):
"""库存管理场景：使用串行化"""
print("\n=== 库存管理场景（串行化）===")
def update_inventory(product_id, quantity_change, operation_type):
"""更新库存"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN EXCLUSIVE TRANSACTION') # 使用串行化
print(f"[库存管理{product_id}] 获取排他锁")
# 检查当前库存
cursor.execute('SELECT name, stock FROM products WHERE id = ?', (product_id,))
product = cursor.fetchone()
print(f"[库存管理{product_id}] 当前库存: {product[0]} = {product[1]}")
if operation_type == 'sale' and product[1] < quantity_change:
print(f"[库存管理{product_id}] ❌ 库存不足")
conn.rollback()
return False
# 更新库存
cursor.execute('UPDATE products SET stock = stock + ? WHERE id = ?', (quantity_change, product_id))
# 记录操作日志
operation_desc = f"销售-{quantity_change}" if operation_type == 'sale' else f"入库-{quantity_change}"
print(f"[库存管理{product_id}] {operation_desc}")
# 模拟处理时间
time.sleep(0.5)
conn.commit()
print(f"[库存管理{product_id}] ✅ 库存更新完成")
return True
except Exception as e:
print(f"[库存管理{product_id}] 异常: {e}")
if 'conn' in locals():
conn.rollback()
return False
finally:
if 'conn' in locals():
conn.close()
def check_inventory():
"""检查库存状态"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('SELECT name, category, stock FROM products ORDER BY id')
products = cursor.fetchall()
print("[库存检查] 当前库存状态:")
for product in products:
print(f" {product[0]} ({product[1]}): {product[2]}")
conn.close()
print("更新前库存状态:")
check_inventory()
# 并发执行库存操作
operations = [
(2, 2, 'sale'), # 销售2台笔记本电脑
(2, 5, 'stock_in'), # 入库5台笔记本电脑
(3, 3, 'sale'), # 销售3双运动鞋
(4, 50, 'stock_in') # 入库50本图书
]
threads = []
for product_id, quantity, op_type in operations:
thread = threading.Thread(
target=update_inventory,
args=(product_id, quantity, op_type)
)
threads.append(thread)
print("\n开始并发库存操作...")
for thread in threads:
thread.start()
for thread in threads:
thread.join()
print("\n更新后库存状态:")
check_inventory()
if __name__ == "__main__":
selector = IsolationLevelSelector()
selector.report_generation_scenario()
selector.inventory_management_scenario()
---
b.性能与一致性权衡
a.性能指标分析
不同隔离级别对性能的影响主要体现在锁争用、事务等待时间和系统吞吐量等方面。需要根据业务需求在性能和数据一致性之间找到最佳平衡点。
b.监控与调优
实时监控数据库的锁等待时间、死锁频率和事务执行时间，根据监控数据动态调整隔离级别和业务逻辑。
c.代码示例
---
# 隔离级别性能监控与优化
import time
import threading
import sqlite3
from collections import defaultdict
from contextlib import contextmanager
class IsolationPerformanceMonitor:
"""隔离级别性能监控器"""
def __init__(self, db_name='isolation_performance_demo.db'):
self.db_name = db_name
self.metrics = defaultdict(list)
self.lock = threading.Lock()
self.setup_database()
def setup_database(self):
"""设置测试环境"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
CREATE TABLE IF NOT EXISTS test_table (
id INTEGER PRIMARY KEY,
data TEXT NOT NULL,
value INTEGER NOT NULL,
timestamp DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('DELETE FROM test_table')
conn.commit()
conn.close()
@contextmanager
def measure_transaction(self, isolation_level, operation_name):
"""事务性能测量上下文管理器"""
start_time = time.time()
start_memory = 0 # 简化的内存监控
try:
yield
finally:
end_time = time.time()
execution_time = end_time - start_time
with self.lock:
self.metrics[f'{isolation_level}_{operation_name}'].append({
'execution_time': execution_time,
'timestamp': end_time
})
def benchmark_isolation_levels(self):
"""基准测试不同隔离级别的性能"""
print("=== 隔离级别性能基准测试 ===")
def read_operation(isolation_level, operation_id):
"""读操作基准测试"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
if isolation_level == 'read_uncommitted':
cursor.execute('PRAGMA read_uncommitted = 1')
cursor.execute('BEGIN TRANSACTION')
with self.measure_transaction(isolation_level, 'read'):
# 模拟复杂查询
cursor.execute('SELECT COUNT(*) FROM test_table')
count = cursor.fetchone()[0]
for i in range(10):
cursor.execute('SELECT data, value FROM test_table ORDER BY id LIMIT 10')
rows = cursor.fetchall()
conn.commit()
except Exception as e:
print(f"[读操作{operation_id}] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def write_operation(isolation_level, operation_id):
"""写操作基准测试"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
if isolation_level == 'serializable':
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
elif isolation_level == 'repeatable_read':
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
else:
cursor.execute('BEGIN TRANSACTION')
with self.measure_transaction(isolation_level, 'write'):
# 模拟批量插入
for i in range(20):
cursor.execute('''
INSERT INTO test_table (data, value) VALUES (?, ?)
''', (f'data_{operation_id}_{i}', i))
conn.commit()
except Exception as e:
print(f"[写操作{operation_id}] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
isolation_levels = ['read_committed', 'read_uncommitted', 'repeatable_read', 'serializable']
for isolation_level in isolation_levels:
print(f"\n测试隔离级别: {isolation_level}")
# 重置数据库
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('DELETE FROM test_table')
conn.commit()
conn.close()
# 并发读写操作
threads = []
# 启动读操作
for i in range(3):
thread = threading.Thread(target=read_operation, args=(isolation_level, i))
threads.append(thread)
# 启动写操作
for i in range(2):
thread = threading.Thread(target=write_operation, args=(isolation_level, i))
threads.append(thread)
# 等待所有操作完成
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# 收集性能指标
time.sleep(0.1) # 确保所有指标都已记录
def generate_performance_report(self):
"""生成性能报告"""
print("\n=== 性能分析报告 ===")
isolation_levels = ['read_committed', 'read_uncommitted', 'repeatable_read', 'serializable']
print("隔离级别性能对比:")
print(f"{'隔离级别':<15} {'平均读时间':<12} {'平均写时间':<12} {'总操作数':<10}")
print("-" * 55)
for iso_level in isolation_levels:
read_times = self.metrics.get(f'{iso_level}_read', [])
write_times = self.metrics.get(f'{iso_level}_write', [])
avg_read_time = sum(m['execution_time'] for m in read_times) / len(read_times) if read_times else 0
avg_write_time = sum(m['execution_time'] for m in write_times) / len(write_times) if write_times else 0
total_ops = len(read_times) + len(write_times)
print(f"{iso_level:<15} {avg_read_time:<12.4f} {avg_write_time:<12.4f} {total_ops:<10}")
# 性能建议
print("\n性能优化建议:")
read_committed_read = self.metrics.get('read_committed_read', [])
read_uncommitted_read = self.metrics.get('read_uncommitted_read', [])
if read_uncommitted_read and read_committed_read:
read_committed_avg = sum(m['execution_time'] for m in read_committed_read) / len(read_committed_read)
read_uncommitted_avg = sum(m['execution_time'] for m in read_uncommitted_read) / len(read_uncommitted_read)
improvement = (read_committed_avg - read_uncommitted_avg) / read_committed_avg * 100
if improvement > 10:
print(f"• 对于读密集型应用，读未提交比读已提交快 {improvement:.1f}%")
print(" 建议：在允许脏读风险的场景下可考虑使用读未提交")
serializable_write = self.metrics.get('serializable_write', [])
read_committed_write = self.metrics.get('read_committed_write', [])
if serializable_write and read_committed_write:
serializable_avg = sum(m['execution_time'] for m in serializable_write) / len(serializable_write)
read_committed_avg = sum(m['execution_time'] for m in read_committed_write) / len(read_committed_write)
overhead = (serializable_avg - read_committed_avg) / read_committed_avg * 100
if overhead > 50:
print(f"• 串行化比读已提交慢 {overhead:.1f}%")
print(" 建议：仅在需要强一致性保证时使用串行化")
def performance_optimization_demo(self):
"""性能优化演示"""
print("\n=== 性能优化策略演示 ===")
def optimized_batch_operation():
"""优化的批量操作"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 启用WAL模式提高并发性能
cursor.execute('PRAGMA journal_mode = WAL')
cursor.execute('PRAGMA synchronous = NORMAL')
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print("[优化操作] 开始批量操作")
start_time = time.time()
# 批量插入优化
batch_data = [(f'batch_data_{i}', i) for i in range(1000)]
cursor.executemany(
'INSERT INTO test_table (data, value) VALUES (?, ?)',
batch_data
)
# 批量更新优化
cursor.execute('UPDATE test_table SET value = value * 2 WHERE value % 2 = 0')
conn.commit()
end_time = time.time()
execution_time = end_time - start_time
print(f"[优化操作] 批量操作完成，耗时: {execution_time:.4f}秒")
except Exception as e:
print(f"[优化操作] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
def standard_operation():
"""标准操作对比"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION')
print("[标准操作] 开始标准操作")
start_time = time.time()
# 标准逐条操作
for i in range(1000):
cursor.execute('INSERT INTO test_table (data, value) VALUES (?, ?)',
(f'standard_data_{i}', i))
conn.commit()
end_time = time.time()
execution_time = end_time - start_time
print(f"[标准操作] 标准操作完成，耗时: {execution_time:.4f}秒")
except Exception as e:
print(f"[标准操作] 异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
# 清理测试数据
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('DELETE FROM test_table')
conn.commit()
conn.close()
# 对比优化效果
optimized_batch_operation()
standard_operation()
if __name__ == "__main__":
monitor = IsolationPerformanceMonitor()
monitor.benchmark_isolation_levels()
monitor.generate_performance_report()
monitor.performance_optimization_demo()
---
04.最佳实践总结
a.隔离级别选择指南
a.业务需求分析
根据业务场景的数据一致性要求、并发性能需求和容错能力来选择合适的隔离级别。关键因素包括：读写比例、数据重要性、性能要求、业务容忍度等。
b.渐进式升级策略
从较低隔离级别开始，根据实际运行中出现的并发问题逐步升级隔离级别，避免过度使用高隔离级别影响性能。
c.代码示例
---
# 隔离级别最佳实践指南
class IsolationBestPractices:
"""隔离级别最佳实践指南"""
def __init__(self, db_name='isolation_best_practices.db'):
self.db_name = db_name
self.setup_database()
def setup_database(self):
"""设置示例数据库"""
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 创建不同类型的业务表
cursor.execute('''
CREATE TABLE IF NOT EXISTS financial_transactions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
transaction_id TEXT NOT NULL UNIQUE,
from_account TEXT NOT NULL,
to_account TEXT NOT NULL,
amount REAL NOT NULL,
status TEXT NOT NULL DEFAULT 'pending',
create_time DATETIME DEFAULT CURRENT_TIMESTAMP,
process_time DATETIME
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS inventory_items (
id INTEGER PRIMARY KEY AUTOINCREMENT,
sku TEXT NOT NULL UNIQUE,
name TEXT NOT NULL,
category TEXT NOT NULL,
current_stock INTEGER NOT NULL DEFAULT 0,
reserved_stock INTEGER NOT NULL DEFAULT 0,
safety_stock INTEGER NOT NULL DEFAULT 0,
last_updated DATETIME DEFAULT CURRENT_TIMESTAMP
)
''')
cursor.execute('''
CREATE TABLE IF NOT EXISTS user_sessions (
id INTEGER PRIMARY KEY AUTOINCREMENT,
user_id INTEGER NOT NULL,
session_token TEXT NOT NULL,
login_time DATETIME DEFAULT CURRENT_TIMESTAMP,
last_activity DATETIME DEFAULT CURRENT_TIMESTAMP,
activity_count INTEGER DEFAULT 0
)
''')
# 初始化示例数据
cursor.execute('DELETE FROM financial_transactions')
cursor.execute('DELETE FROM inventory_items')
cursor.execute('DELETE FROM user_sessions')
# 初始化库存
inventory_data = [
('LAPTOP001', 'MacBook Pro 14"', '电子产品', 50, 5, 10),
('MOUSE001', '无线鼠标', '电子产品', 200, 10, 20),
('KEYBOARD001', '机械键盘', '电子产品', 100, 8, 15),
('MONITOR001', '27寸显示器', '电子产品', 75, 3, 12)
]
cursor.executemany(
'INSERT INTO inventory_items (sku, name, category, current_stock, reserved_stock, safety_stock) VALUES (?, ?, ?, ?, ?, ?)',
inventory_data
)
conn.commit()
conn.close()
print("示例数据库初始化完成")
def financial_transaction_scenario(self):
"""金融交易场景：使用串行化隔离级别"""
print("=== 金融交易场景（串行化隔离）===")
def process_financial_transaction(tx_id, from_acc, to_acc, amount):
"""处理金融交易"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 使用串行化隔离级别确保金融交易的一致性
cursor.execute('BEGIN EXCLUSIVE TRANSACTION')
print(f"[交易{tx_id}] 开始处理，获取排他锁")
# 验证交易有效性
if amount <= 0:
raise ValueError("交易金额必须大于0")
# 检查账户余额（简化逻辑，实际应用中需要查询账户表）
print(f"[交易{tx_id}] 验证交易：{from_acc} → {to_acc}, 金额：{amount}")
# 模拟账户余额检查
# 这里假设有足够的余额
# 记录交易
cursor.execute('''
INSERT INTO financial_transactions (transaction_id, from_account, to_account, amount, status, process_time)
VALUES (?, ?, ?, ?, ?, ?)
''', (tx_id, from_acc, to_acc, amount, 'completed', datetime.now()))
# 模拟外部系统调用
time.sleep(0.5)
conn.commit()
print(f"[交易{tx_id}] ✅ 交易处理完成")
return True
except Exception as e:
print(f"[交易{tx_id}] ❌ 交易失败: {e}")
if 'conn' in locals():
conn.rollback()
return False
finally:
if 'conn' in locals():
conn.close()
def audit_transaction(tx_id):
"""审计交易"""
time.sleep(0.1) # 延迟启动审计
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION') # 读已提交足够
# 查询交易详情
cursor.execute('''
SELECT transaction_id, from_account, to_account, amount, status, process_time
FROM financial_transactions
WHERE transaction_id = ?
''', (tx_id,))
transaction = cursor.fetchone()
if transaction:
print(f"[审计{tx_id}] 交易详情: {transaction[0]} {transaction[1]}→{transaction[2]} {transaction[3]} {transaction[4]} {transaction[5]}")
else:
print(f"[审计{tx_id}] 交易不存在")
conn.commit()
except Exception as e:
print(f"[审计{tx_id}] 审计异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
from datetime import datetime
# 并发处理多个金融交易
transactions = [
('TXN001', 'ACC1001', 'ACC2001', 10000.0),
('TXN002', 'ACC1002', 'ACC2002', 5000.0),
('TXN003', 'ACC1003', 'ACC2003', 15000.0)
]
threads = []
for tx_id, from_acc, to_acc, amount in transactions:
# 交易处理线程
tx_thread = threading.Thread(
target=process_financial_transaction,
args=(tx_id, from_acc, to_acc, amount)
)
threads.append(tx_thread)
# 审计线程
audit_thread = threading.Thread(target=audit_transaction, args=(tx_id,))
threads.append(audit_thread)
print("开始并发金融交易处理...")
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# 查看交易结果
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('SELECT transaction_id, status FROM financial_transactions ORDER BY create_time')
results = cursor.fetchall()
print("\n交易处理结果:")
for result in results:
status_icon = "✅" if result[1] == 'completed' else "❌"
print(f" {result[0]}: {status_icon} {result[1]}")
conn.close()
def inventory_management_scenario(self):
"""库存管理场景：使用可重复读隔离级别"""
print("\n=== 库存管理场景（可重复读隔离）===")
def update_inventory(sku, quantity_change, operation_type, reservation_id=None):
"""更新库存"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 使用可重复读隔离级别，确保库存读取的一致性
cursor.execute('BEGIN IMMEDIATE TRANSACTION')
print(f"[库存{sku}] 开始{operation_type}操作")
# 读取当前库存信息（可重复读确保一致性）
cursor.execute('''
SELECT current_stock, reserved_stock, safety_stock, name
FROM inventory_items
WHERE sku = ?
''', (sku,))
result = cursor.fetchone()
if not result:
raise ValueError(f"商品 {sku} 不存在")
current_stock, reserved_stock, safety_stock, name = result
available_stock = current_stock - reserved_stock
print(f"[库存{sku}] 当前状态: 总库存={current_stock}, 预留={reserved_stock}, 可用={available_stock}")
if operation_type == 'reserve':
if quantity_change > available_stock:
raise ValueError(f"库存不足：可用{available_stock}, 需要{quantity_change}")
# 增加预留库存
cursor.execute('''
UPDATE inventory_items
SET reserved_stock = reserved_stock + ?, last_updated = ?
WHERE sku = ?
''', (quantity_change, datetime.now(), sku))
print(f"[库存{sku}] 预留{quantity_change}个 {name}")
elif operation_type == 'release':
# 释放预留库存
if quantity_change > reserved_stock:
raise ValueError(f"预留库存不足：预留{reserved_stock}, 释放{quantity_change}")
cursor.execute('''
UPDATE inventory_items
SET reserved_stock = reserved_stock - ?, last_updated = ?
WHERE sku = ?
''', (quantity_change, datetime.now(), sku))
print(f"[库存{sku}] 释放预留{quantity_change}个 {name}")
elif operation_type == 'confirm':
# 确认出库，同时减少总库存和预留库存
if quantity_change > reserved_stock:
raise ValueError(f"预留库存不足：预留{reserved_stock}, 确认{quantity_change}")
cursor.execute('''
UPDATE inventory_items
SET current_stock = current_stock - ?,
reserved_stock = reserved_stock - ?,
last_updated = ?
WHERE sku = ?
''', (quantity_change, quantity_change, datetime.now(), sku))
print(f"[库存{sku}] 确认出库{quantity_change}个 {name}")
# 检查安全库存
cursor.execute('SELECT current_stock, safety_stock FROM inventory_items WHERE sku = ?', (sku,))
stock_info = cursor.fetchone()
if stock_info[0] < stock_info[1]:
print(f"[库存{sku}] ⚠️ 库存低于安全库存：当前{stock_info[0]}, 安全线{stock_info[1]}")
conn.commit()
print(f"[库存{sku}] ✅ {operation_type}操作完成")
return True
except Exception as e:
print(f"[库存{sku}] ❌ {operation_type}操作失败: {e}")
if 'conn' in locals():
conn.rollback()
return False
finally:
if 'conn' in locals():
conn.close()
from datetime import datetime
# 模拟电商订单处理流程
print("开始模拟电商订单处理流程...")
# 并发操作
operations = [
('LAPTOP001', 3, 'reserve'), # 预留3台笔记本电脑
('MOUSE001', 15, 'reserve'), # 预留15个鼠标
('KEYBOARD001', 8, 'reserve'), # 预留8个键盘
('LAPTOP001', 2, 'confirm'), # 确认2台笔记本电脑出库
('MOUSE001', 10, 'confirm'), # 确认10个鼠标出库
('LAPTOP001', 1, 'release') # 释放1台笔记本电脑预留
]
threads = []
for i, (sku, qty, op_type) in enumerate(operations):
thread = threading.Thread(
target=update_inventory,
args=(sku, qty, op_type, f'RES_{i+1}')
)
threads.append(thread)
# 启动所有操作
for thread in threads:
thread.start()
for thread in threads:
thread.join()
# 查看最终库存状态
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('''
SELECT sku, name, current_stock, reserved_stock, safety_stock,
(current_stock - reserved_stock) as available
FROM inventory_items
ORDER BY sku
''')
print("\n最终库存状态:")
print(f"{'SKU':<12} {'名称':<15} {'总库存':<8} {'预留':<8} {'可用':<8} {'安全库存':<8}")
print("-" * 70)
for row in cursor.fetchall():
sku, name, total, reserved, safety, available = row
warning = "⚠️" if available < safety else "✅"
print(f"{sku:<12} {name:<15} {total:<8} {reserved:<8} {available:<8} {safety:<8} {warning}")
conn.close()
def user_activity_tracking_scenario(self):
"""用户活动跟踪场景：使用读已提交隔离级别"""
print("\n=== 用户活动跟踪场景（读已提交隔离）===")
def track_user_activity(user_id, activity_count):
"""跟踪用户活动"""
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
# 使用读已提交隔离级别，适合高并发的用户活动记录
cursor.execute('BEGIN TRANSACTION') # 默认读已提交
# 检查是否有活跃会话
cursor.execute('''
SELECT id, activity_count FROM user_sessions
WHERE user_id = ?
ORDER BY last_activity DESC
LIMIT 1
''', (user_id,))
session = cursor.fetchone()
if session and (datetime.now() - datetime.strptime(session[2].split('.')[0], '%Y-%m-%d %H:%M:%S')).seconds < 1800:
# 更新现有会话
session_id = session[0]
new_count = session[1] + activity_count
cursor.execute('''
UPDATE user_sessions
SET activity_count = ?, last_activity = ?
WHERE id = ?
''', (new_count, datetime.now(), session_id))
print(f"[用户{user_id}] 更新会话{session_id}，活动次数: {new_count}")
else:
# 创建新会话
session_token = f"token_{user_id}_{int(time.time())}"
cursor.execute('''
INSERT INTO user_sessions (user_id, session_token, activity_count, last_activity)
VALUES (?, ?, ?, ?)
''', (user_id, session_token, activity_count, datetime.now()))
session_id = cursor.lastrowid
print(f"[用户{user_id}] 创建新会话{session_id}，活动次数: {activity_count}")
conn.commit()
return session_id
except Exception as e:
print(f"[用户{user_id}] 活动跟踪异常: {e}")
if 'conn' in locals():
conn.rollback()
return None
finally:
if 'conn' in locals():
conn.close()
def generate_user_report():
"""生成用户活动报告"""
time.sleep(2) # 延迟生成报告
try:
conn = sqlite3.connect(self.db_name)
cursor = conn.cursor()
cursor.execute('BEGIN TRANSACTION')
# 统计活跃用户
cursor.execute('SELECT COUNT(DISTINCT user_id) FROM user_sessions')
active_users = cursor.fetchone()[0]
# 统计总活动次数
cursor.execute('SELECT SUM(activity_count) FROM user_sessions')
total_activities = cursor.fetchone()[0] or 0
# 按用户统计
cursor.execute('''
SELECT user_id, COUNT(*) as session_count, SUM(activity_count) as total_activities
FROM user_sessions
GROUP BY user_id
ORDER BY total_activities DESC
LIMIT 5
''')
top_users = cursor.fetchall()
print(f"[报告] 活跃用户数: {active_users}")
print(f"[报告] 总活动次数: {total_activities}")
print("[报告] 活跃用户TOP5:")
for user_data in top_users:
print(f" 用户{user_data[0]}: {user_data[1]}个会话, {user_data[2]}次活动")
conn.commit()
except Exception as e:
print(f"[报告] 生成异常: {e}")
if 'conn' in locals():
conn.rollback()
finally:
if 'conn' in locals():
conn.close()
from datetime import datetime
# 模拟多用户并发活动
print("开始模拟用户并发活动...")
user_activities = [
(1001, 5), (1002, 3), (1003, 8), (1001, 2), (1004, 6),
(1002, 4), (1005, 1), (1003, 7), (1001, 3), (1004, 2)
]
threads = []
# 启动用户活动跟踪
for user_id, activity_count in user_activities:
thread = threading.Thread(
target=track_user_activity,
args=(user_id, activity_count)
)
threads.append(thread)
# 启动报告生成
report_thread = threading.Thread(target=generate_user_report)
threads.append(report_thread)
for thread in threads:
thread.start()
for thread in threads:
thread.join()
if __name__ == "__main__":
practices = IsolationBestPractices()
practices.financial_transaction_scenario()
practices.inventory_management_scenario()
practices.user_activity_tracking_scenario()
---
3. MySQL事务
3.1 pymysql连接器
01.pymysql概述
a.基本介绍
pymysql是Python语言编写的MySQL数据库连接器，提供纯Python实现的MySQL客户端协议，支持MySQL和MariaDB数据库。
b.核心特性
a.功能说明
支持完整的MySQL协议，包括事务控制、预处理语句、存储过程调用等功能，兼容Python 2.7+和3.x版本。
b.技术优势
纯Python实现无需编译，易于安装和部署，支持SSL连接和压缩传输。
02.pymysql安装与配置
a.安装方法
a.功能说明
通过pip包管理器安装pymysql模块，支持从PyPI仓库自动下载依赖。
b.代码示例
---
# 安装pymysql
pip install pymysql
# 验证安装
import pymysql
print(pymysql.__version__)
---
b.连接配置
a.功能说明
配置数据库连接参数，包括主机地址、端口、用户名、密码、数据库名等基本信息。
b.代码示例
---
# 基本连接配置
import pymysql
# 连接参数字典
config = {
'host': 'localhost', # 数据库主机地址
'port': 3306, # 端口号
'user': 'root', # 用户名
'password': 'password', # 密码
'database': 'test_db', # 数据库名
'charset': 'utf8mb4', # 字符集
'cursorclass': pymysql.cursors.DictCursor # 返回字典格式
}
# 建立连接
connection = pymysql.Connect(**config)
print("数据库连接成功")
connection.close()
---
03.pymysql连接管理
a.连接对象操作
a.功能说明
Connection对象提供数据库连接的核心功能，包括事务控制、游标创建、连接状态检查等。
b.代码示例
---
import pymysql
# 创建连接
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='test_db',
charset='utf8mb4'
)
# 检查连接状态
print(f"连接是否开启: {conn.open}")
print(f"服务器信息: {conn.get_server_info()}")
print(f"连接ID: {conn.thread_id()}")
# 获取数据库信息
cursor = conn.cursor()
cursor.execute("SELECT DATABASE()")
current_db = cursor.fetchone()
print(f"当前数据库: {current_db}")
cursor.close()
conn.close()
---
b.连接池管理
a.功能说明
实现数据库连接池，提高连接复用率，减少频繁创建和销毁连接的开销。
b.代码示例
---
import pymysql
from threading import local
class ConnectionPool:
def __init__(self, max_connections=10, **config):
self.max_connections = max_connections
self.config = config
self.pool = []
self.local = local()
def get_connection(self):
if not hasattr(self.local, 'connection'):
if self.pool:
conn = self.pool.pop()
else:
conn = pymysql.Connect(**self.config)
self.local.connection = conn
return self.local.connection
def release_connection(self):
if hasattr(self.local, 'connection'):
conn = self.local.connection
if len(self.pool) < self.max_connections:
self.pool.append(conn)
del self.local.connection
def close_all(self):
for conn in self.pool:
conn.close()
self.pool.clear()
# 使用连接池
pool_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'test_db',
'charset': 'utf8mb4'
}
pool = ConnectionPool(max_connections=5, **pool_config)
# 获取连接并使用
conn = pool.get_connection()
cursor = conn.cursor()
cursor.execute("SELECT 1")
result = cursor.fetchone()
print(f"查询结果: {result}")
cursor.close()
# 释放连接
pool.release_connection()
pool.close_all()
---
04.pymysql事务基础
a.自动提交模式
a.功能说明
默认情况下，pymysql启用自动提交模式，每条SQL语句作为一个独立事务执行。
b.代码示例
---
import pymysql
# 自动提交模式演示
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='test_db',
charset='utf8mb4',
autocommit=True # 启用自动提交
)
cursor = conn.cursor()
# 创建测试表
cursor.execute("""
CREATE TABLE IF NOT EXISTS test_transaction (
id INT PRIMARY KEY AUTO_INCREMENT,
name VARCHAR(50),
amount INT
)
""")
# 插入数据 - 自动提交
cursor.execute("INSERT INTO test_transaction (name, amount) VALUES (%s, %s)",
("Alice", 100))
print(f"影响的行数: {cursor.rowcount}")
# 查询数据
cursor.execute("SELECT * FROM test_transaction")
results = cursor.fetchall()
print(f"查询结果: {results}")
cursor.close()
conn.close()
---
b.手动事务控制
a.功能说明
禁用自动提交后，需要手动调用commit()提交事务或rollback()回滚事务。
b.代码示例
---
import pymysql
# 手动事务控制演示
conn = pymysql.Connect(
host='localhost',
user='root',
'password': 'password',
database='test_db',
charset='utf8mb4',
autocommit=False # 禁用自动提交
)
cursor = conn.cursor()
try:
# 开始事务（默认已开始）
# 删除测试数据
cursor.execute("DELETE FROM test_transaction")
# 插入多条数据
insert_data = [
("Alice", 100),
("Bob", 200),
("Charlie", 300)
]
cursor.executemany(
"INSERT INTO test_transaction (name, amount) VALUES (%s, %s)",
insert_data
)
print(f"插入成功，影响的行数: {cursor.rowcount}")
# 查询中间状态
cursor.execute("SELECT COUNT(*) FROM test_transaction")
count = cursor.fetchone()[0]
print(f"当前记录数: {count}")
# 手动提交事务
conn.commit()
print("事务提交成功")
except Exception as e:
# 发生错误时回滚事务
conn.rollback()
print(f"事务回滚: {e}")
finally:
cursor.close()
conn.close()
---
05.pymysql连接器最佳实践
a.连接安全配置
a.功能说明
配置SSL连接和密码加密，确保数据传输安全，防止敏感信息泄露。
b.代码示例
---
import pymysql
# SSL安全连接配置
ssl_config = {
'host': 'your-mysql-server.com',
'port': 3306,
'user': 'secure_user',
'password': 'secure_password',
'database': 'secure_db',
'charset': 'utf8mb4',
'ssl': {
'ca': '/path/to/ca.pem',
'cert': '/path/to/client-cert.pem',
'key': '/path/to/client-key.pem'
},
'connect_timeout': 10,
'read_timeout': 30,
'write_timeout': 30
}
try:
# 建立SSL连接
conn = pymysql.Connect(**ssl_config)
print("安全连接建立成功")
# 验证SSL状态
cursor = conn.cursor()
cursor.execute("SHOW STATUS LIKE 'Ssl_version'")
ssl_version = cursor.fetchone()
print(f"SSL版本: {ssl_version}")
cursor.close()
conn.close()
except pymysql.Error as e:
print(f"安全连接失败: {e}")
---
b.错误处理与重试
a.功能说明
实现连接超时处理、网络中断重试、死锁检测等异常情况的自动恢复机制。
b.代码示例
---
import pymysql
import time
import random
def execute_with_retry(sql, params=None, max_retries=3, retry_delay=1):
"""带重试机制的SQL执行函数"""
for attempt in range(max_retries):
conn = None
try:
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='test_db',
charset='utf8mb4',
autocommit=False,
connect_timeout=5,
read_timeout=10,
write_timeout=10
)
cursor = conn.cursor()
cursor.execute(sql, params or ())
if sql.strip().upper().startswith(('SELECT', 'SHOW')):
result = cursor.fetchall()
else:
result = cursor.rowcount
conn.commit()
return result
except pymysql.OperationalError as e:
if conn:
try:
conn.rollback()
except:
pass
if 'deadlock' in str(e).lower() and attempt < max_retries - 1:
# 死锁时随机延迟重试
delay = retry_delay * (2 ** attempt) + random.uniform(0, 1)
print(f"检测到死锁，{delay:.1f}秒后重试 (尝试 {attempt + 1}/{max_retries})")
time.sleep(delay)
continue
else:
raise e
except Exception as e:
if conn:
try:
conn.rollback()
except:
pass
raise e
finally:
if conn:
conn.close()
raise Exception(f"执行失败，已重试 {max_retries} 次")
# 使用示例
try:
# 执行查询
results = execute_with_retry("SELECT * FROM test_transaction WHERE amount > %s", (150,))
print(f"查询结果: {results}")
# 执行更新
affected = execute_with_retry("UPDATE test_transaction SET amount = amount + 10 WHERE name = %s", ("Alice",))
print(f"更新的行数: {affected}")
except Exception as e:
print(f"执行失败: {e}")
---
3.2 事务控制
01.事务生命周期管理
a.事务开始
a.功能说明
在pymysql中，通过设置autocommit=False来启动事务模式，此时事务默认开始，直到执行commit()或rollback()。
b.代码示例
---
import pymysql
import logging
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def create_transaction_connection():
"""创建事务连接"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False, # 禁用自动提交，开启事务模式
connect_timeout=10
)
logger.info("事务连接已建立，autocommit=False")
return conn
def setup_database():
"""初始化数据库和表"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
charset='utf8mb4'
)
cursor = conn.cursor()
try:
# 创建数据库
cursor.execute("CREATE DATABASE IF NOT EXISTS bank_db")
cursor.execute("USE bank_db")
# 创建账户表
cursor.execute("""
CREATE TABLE IF NOT EXISTS accounts (
id INT PRIMARY KEY AUTO_INCREMENT,
account_number VARCHAR(20) UNIQUE NOT NULL,
account_name VARCHAR(100) NOT NULL,
balance DECIMAL(15,2) DEFAULT 0.00,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP
)
""")
# 创建交易记录表
cursor.execute("""
CREATE TABLE IF NOT EXISTS transactions (
id INT PRIMARY KEY AUTO_INCREMENT,
from_account VARCHAR(20) NOT NULL,
to_account VARCHAR(20) NOT NULL,
amount DECIMAL(15,2) NOT NULL,
transaction_type VARCHAR(20) NOT NULL,
status VARCHAR(20) DEFAULT 'pending',
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
processed_at TIMESTAMP NULL
)
""")
conn.commit()
logger.info("数据库和表创建成功")
except Exception as e:
conn.rollback()
logger.error(f"数据库初始化失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 初始化数据库
setup_database()
---
b.事务提交
a.功能说明
使用commit()方法永久保存事务中的所有更改，使更改对其他连接可见，并释放事务锁定的资源。
b.代码示例
---
import pymysql
import logging
logger = logging.getLogger(__name__)
def deposit_money(account_number, amount):
"""存款操作 - 事务提交示例"""
conn = None
try:
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
logger.info(f"开始存款操作: 账户 {account_number}, 金额 {amount}")
# 1. 检查账户是否存在
cursor.execute("SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(account_number,))
account = cursor.fetchone()
if not account:
raise Exception(f"账户 {account_number} 不存在")
account_id, current_balance = account
logger.info(f"账户当前余额: {current_balance}")
# 2. 更新账户余额
new_balance = float(current_balance) + float(amount)
cursor.execute(
"UPDATE accounts SET balance = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
(new_balance, account_id)
)
# 3. 记录交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
(account_number, account_number, amount, 'DEPOSIT', 'completed')
)
# 4. 提交事务 - 所有操作永久生效
conn.commit()
logger.info(f"存款成功: 新余额 {new_balance}")
return new_balance
except Exception as e:
if conn:
conn.rollback()
logger.error(f"存款失败，事务已回滚: {e}")
raise
finally:
if conn:
conn.close()
# 使用示例
try:
new_balance = deposit_money("1000001", 1000.00)
print(f"存款成功，新余额: {new_balance}")
except Exception as e:
print(f"存款失败: {e}")
---
c.事务回滚
a.功能说明
使用rollback()方法撤销当前事务中的所有更改，释放锁定的资源，将数据库状态恢复到事务开始前的状态。
b.代码示例
---
import pymysql
import logging
logger = logging.getLogger(__name__)
def transfer_with_validation(from_account, to_account, amount):
"""转账操作 - 带完整验证的事务回滚示例"""
conn = None
try:
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
logger.info(f"开始转账: {from_account} -> {to_account}, 金额: {amount}")
# 1. 查询并锁定两个账户
cursor.execute("SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(from_account,))
from_acct = cursor.fetchone()
cursor.execute("SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(to_account,))
to_acct = cursor.fetchone()
# 2. 验证账户存在
if not from_acct:
raise Exception(f"转出账户 {from_account} 不存在")
if not to_acct:
raise Exception(f"转入账户 {to_account} 不存在")
from_id, from_balance = from_acct
to_id, to_balance = to_acct
logger.info(f"转出账户余额: {from_balance}, 转入账户余额: {to_balance}")
# 3. 检查余额是否充足
if float(from_balance) < float(amount):
raise Exception(f"转出账户余额不足。当前余额: {from_balance}, 转账金额: {amount}")
# 4. 检查转账金额是否合法
if float(amount) <= 0:
raise Exception(f"转账金额必须大于0，当前金额: {amount}")
# 5. 模拟业务规则检查（单日转账限额）
cursor.execute(
"""SELECT COALESCE(SUM(amount), 0) as daily_total
FROM transactions
WHERE from_account = %s
AND DATE(created_at) = CURDATE()
AND status = 'completed'""",
(from_account,)
)
daily_total = cursor.fetchone()[0]
daily_limit = 100000.00 # 单日限额10万
if float(daily_total) + float(amount) > daily_limit:
raise Exception(f"超过单日转账限额。今日已转账: {daily_total}, 限额: {daily_limit}")
logger.info(f"业务验证通过，开始执行转账操作")
# 6. 记录交易记录
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status)
VALUES (%s, %s, %s, %s, %s)""",
(from_account, to_account, amount, 'TRANSFER', 'processing')
)
transaction_id = cursor.lastrowid
# 7. 执行账户余额更新
new_from_balance = float(from_balance) - float(amount)
new_to_balance = float(to_balance) + float(amount)
cursor.execute(
"UPDATE accounts SET balance = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
(new_from_balance, from_id)
)
cursor.execute(
"UPDATE accounts SET balance = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s",
(new_to_balance, to_id)
)
# 8. 更新交易状态
cursor.execute(
"UPDATE transactions SET status = 'completed', processed_at = CURRENT_TIMESTAMP WHERE id = %s",
(transaction_id,)
)
# 9. 模拟一个可能的错误（用于测试回滚）
if float(amount) > 50000: # 大额转账需要额外验证
raise Exception("大额转账需要人工审核，系统自动拒绝")
# 10. 所有操作成功，提交事务
conn.commit()
logger.info(f"转账成功: 交易ID {transaction_id}, 转出账户新余额: {new_from_balance}")
return transaction_id, new_from_balance
except Exception as e:
if conn:
# 任何错误都会触发回滚
conn.rollback()
logger.warning(f"转账失败，事务已回滚: {e}")
logger.info("所有账户余额已恢复到转账前状态")
raise
finally:
if conn:
conn.close()
# 测试转账功能
try:
# 创建测试账户
test_conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
test_cursor = test_conn.cursor()
# 插入测试数据
test_cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[("1000001", "张三", 10000.00), ("1000002", "李四", 5000.00)]
)
test_conn.commit()
test_cursor.close()
test_conn.close()
# 测试正常转账
result = transfer_with_validation("1000001", "1000002", 1000.00)
print(f"转账成功: {result}")
# 测试触发了回滚的情况（余额不足）
try:
result = transfer_with_validation("1000002", "1000001", 10000.00)
except Exception as e:
print(f"预期的转账失败: {e}")
except Exception as e:
print(f"测试失败: {e}")
---
02.事务隔离级别控制
a.隔离级别介绍
a.功能说明
MySQL支持四种事务隔离级别：READ UNCOMMITTED、READ COMMITTED、REPEATABLE READ、SERIALIZABLE，不同级别提供不同的一致性保证。
b.隔离级别对比
READ UNCOMMITTED：最低级别，允许读取未提交的数据，可能出现脏读
READ COMMITTED：只允许读取已提交的数据，避免脏读，但可能出现不可重复读
REPEATABLE READ：默认级别，确保在同一事务中多次读取同一数据结果一致，避免不可重复读
SERIALIZABLE：最高级别，完全串行化执行，避免所有并发问题，但性能最低
b.隔离级别设置
a.功能说明
在pymysql中可以通过SET SESSION TRANSACTION ISOLATION LEVEL语句设置事务隔离级别，控制并发事务之间的可见性。
b.代码示例
---
import pymysql
import threading
import time
import logging
logger = logging.getLogger(__name__)
def test_isolation_levels():
"""测试不同隔离级别的行为"""
def setup_test_data():
"""准备测试数据"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
# 清理并插入测试数据
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'ISO%'")
cursor.execute(
"INSERT INTO accounts (account_number, account_name, balance) VALUES (%s, %s, %s)",
("ISO001", "隔离测试账户1", 1000.00)
)
conn.commit()
logger.info("测试数据准备完成")
except Exception as e:
conn.rollback()
raise
finally:
cursor.close()
conn.close()
def read_uncommitted_demo():
"""READ UNCOMMITTED 隔离级别演示"""
conn1 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
conn2 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
try:
cursor1 = conn1.cursor()
cursor2 = conn2.cursor()
# 设置隔离级别
cursor1.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
cursor2.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ UNCOMMITTED")
logger.info("=== READ UNCOMMITTED 隔离级别测试 ===")
# 连接1开始事务并修改数据但不提交
conn1.begin()
cursor1.execute(
"UPDATE accounts SET balance = 1500.00 WHERE account_number = %s",
("ISO001",)
)
logger.info("连接1: 已更新余额为1500，但未提交")
# 连接2读取数据（能读到未提交的数据）
conn2.begin()
cursor2.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance = cursor2.fetchone()[0]
logger.info(f"连接2: 读取到余额 {balance} (脏读)")
# 连接1回滚
conn1.rollback()
logger.info("连接1: 事务已回滚")
# 连接2再次读取
cursor2.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance = cursor2.fetchone()[0]
logger.info(f"连接2: 再次读取余额 {balance}")
conn2.commit()
except Exception as e:
logger.error(f"READ UNCOMMITTED 测试失败: {e}")
conn1.rollback()
conn2.rollback()
finally:
cursor1.close()
cursor2.close()
conn1.close()
conn2.close()
def read_committed_demo():
"""READ COMMITTED 隔离级别演示"""
conn1 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
conn2 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
try:
cursor1 = conn1.cursor()
cursor2 = conn2.cursor()
# 设置隔离级别
cursor1.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED")
cursor2.execute("SET SESSION TRANSACTION ISOLATION LEVEL READ COMMITTED")
logger.info("\n=== READ COMMITTED 隔离级别测试 ===")
# 连接1开始事务
conn1.begin()
cursor1.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance1 = cursor1.fetchone()[0]
logger.info(f"连接1: 第一次读取余额 {balance1}")
# 连接2修改并提交
conn2.begin()
cursor2.execute(
"UPDATE accounts SET balance = 2000.00 WHERE account_number = %s",
("ISO001",)
)
conn2.commit()
logger.info("连接2: 已更新余额为2000并提交")
# 连接1再次读取（能读到其他事务已提交的更改）
cursor1.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance2 = cursor1.fetchone()[0]
logger.info(f"连接1: 第二次读取余额 {balance2} (不可重复读)")
conn1.commit()
except Exception as e:
logger.error(f"READ COMMITTED 测试失败: {e}")
conn1.rollback()
conn2.rollback()
finally:
cursor1.close()
cursor2.close()
conn1.close()
conn2.close()
def repeatable_read_demo():
"""REPEATABLE READ 隔离级别演示"""
conn1 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
conn2 = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
try:
cursor1 = conn1.cursor()
cursor2 = conn2.cursor()
# 设置隔离级别（这是MySQL的默认级别）
cursor1.execute("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ")
cursor2.execute("SET SESSION TRANSACTION ISOLATION LEVEL REPEATABLE READ")
logger.info("\n=== REPEATABLE READ 隔离级别测试 ===")
# 连接1开始事务并读取
conn1.begin()
cursor1.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance1 = cursor1.fetchone()[0]
logger.info(f"连接1: 第一次读取余额 {balance1}")
# 连接2修改并提交
conn2.begin()
cursor2.execute(
"UPDATE accounts SET balance = 3000.00 WHERE account_number = %s",
("ISO001",)
)
conn2.commit()
logger.info("连接2: 已更新余额为3000并提交")
# 连接1再次读取（仍然读取到事务开始时的值）
cursor1.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
("ISO001",)
)
balance2 = cursor1.fetchone()[0]
logger.info(f"连接1: 第二次读取余额 {balance2} (可重复读)")
conn1.commit()
except Exception as e:
logger.error(f"REPEATABLE READ 测试失败: {e}")
conn1.rollback()
conn2.rollback()
finally:
cursor1.close()
cursor2.close()
conn1.close()
conn2.close()
try:
setup_test_data()
read_uncommitted_demo()
read_committed_demo()
repeatable_read_demo()
except Exception as e:
logger.error(f"隔离级别测试失败: {e}")
# 运行测试
test_isolation_levels()
---
3.3 保存点Savepoint
01.保存点基本概念
a.定义与作用
a.功能说明
保存点是事务中的一个标记点，允许事务回滚到指定的保存点而不是回滚整个事务，实现部分回滚功能。
b.应用场景
复杂业务处理中的分段错误恢复，批量操作中的错误处理，需要撤销部分操作的场景。
b.保存点操作命令
a.功能说明
主要包括SAVEPOINT设置保存点、ROLLBACK TO SAVEPOINT回滚到保存点、RELEASE SAVEPOINT释放保存点三种操作。
b.语法说明
SAVEPOINT savepoint_name - 设置保存点
ROLLBACK TO SAVEPOINT savepoint_name - 回滚到指定保存点
RELEASE SAVEPOINT savepoint_name - 释放保存点
02.保存点基础操作
a.设置保存点
a.功能说明
使用SAVEPOINT语句在事务中创建标记点，后续可以回滚到此点，保持此点之前的所有更改。
b.代码示例
---
import pymysql
import logging
logger = logging.getLogger(__name__)
def basic_savepoint_demo():
"""基础保存点操作演示"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
logger.info("=== 基础保存点演示 ===")
# 开始事务
conn.begin()
# 创建测试账户
cursor.execute(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
("SP001", "保存点测试账户1", 5000.00)
)
account1_id = cursor.lastrowid
logger.info(f"创建账户1: ID={account1_id}")
# 设置第一个保存点
cursor.execute("SAVEPOINT sp_after_account1")
logger.info("设置保存点: sp_after_account1")
# 创建第二个账户
cursor.execute(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
("SP002", "保存点测试账户2", 3000.00)
)
account2_id = cursor.lastrowid
logger.info(f"创建账户2: ID={account2_id}")
# 设置第二个保存点
cursor.execute("SAVEPOINT sp_after_account2")
logger.info("设置保存点: sp_after_account2")
# 执行一些更新操作
cursor.execute(
"UPDATE accounts SET balance = balance + 1000 WHERE id = %s",
(account1_id,)
)
logger.info("账户1余额增加1000")
# 模拟一个错误操作（比如余额变为负数）
cursor.execute(
"UPDATE accounts SET balance = balance - 10000 WHERE id = %s",
(account2_id,)
)
logger.info("账户2余额减少10000（这会导致负数）")
# 检查是否有负余额
cursor.execute("SELECT id, balance FROM accounts WHERE balance < 0")
negative_accounts = cursor.fetchall()
if negative_accounts:
logger.warning(f"检测到负余额账户: {negative_accounts}")
# 回滚到第二个保存点（撤销更新操作）
cursor.execute("ROLLBACK TO SAVEPOINT sp_after_account2")
logger.info("回滚到保存点 sp_after_account2")
# 重新检查
cursor.execute("SELECT id, balance FROM accounts WHERE balance < 0")
negative_accounts = cursor.fetchall()
if negative_accounts:
logger.error("仍然存在负余额，回滚到第一个保存点")
cursor.execute("ROLLBACK TO SAVEPOINT sp_after_account1")
else:
logger.info("负余额问题已解决")
# 查询当前状态
cursor.execute("SELECT id, account_number, balance FROM accounts WHERE account_number LIKE 'SP%'")
accounts = cursor.fetchall()
logger.info(f"当前账户状态: {accounts}")
# 提交事务
conn.commit()
logger.info("事务提交成功")
return accounts
except Exception as e:
conn.rollback()
logger.error(f"保存点演示失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 运行演示
try:
accounts = basic_savepoint_demo()
print(f"保存点演示完成，创建的账户: {accounts}")
except Exception as e:
print(f"保存点演示失败: {e}")
---
b.回滚到保存点
a.功能说明
使用ROLLBACK TO SAVEPOINT语句撤销事务中指定保存点之后的所有更改，但保留保存点之前的更改。
b.代码示例
---
import pymysql
import logging
logger = logging.getLogger(__name__)
def rollback_to_savepoint_demo():
"""回滚到保存点演示"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
logger.info("=== 回滚到保存点演示 ===")
# 准备测试数据
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'ROLLBACK%'")
cursor.execute(
"INSERT INTO accounts (account_number, account_name, balance) VALUES (%s, %s, %s)",
("ROLLBACK001", "回滚测试账户", 10000.00)
)
cursor.execute("SELECT id FROM accounts WHERE account_number = %s", ("ROLLBACK001",))
account_id = cursor.fetchone()[0]
conn.commit()
# 开始新事务
conn.begin()
logger.info(f"开始事务，账户ID: {account_id}")
# 记录初始余额
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
initial_balance = cursor.fetchone()[0]
logger.info(f"初始余额: {initial_balance}")
# 操作1: 存款
cursor.execute(
"UPDATE accounts SET balance = balance + 2000 WHERE id = %s",
(account_id,)
)
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
balance_after_deposit = cursor.fetchone()[0]
logger.info(f"存款后余额: {balance_after_deposit}")
# 设置保存点1
cursor.execute("SAVEPOINT sp_after_deposit")
logger.info("设置保存点: sp_after_deposit")
# 操作2: 转账支出
cursor.execute(
"UPDATE accounts SET balance = balance - 1500 WHERE id = %s",
(account_id,)
)
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
balance_after_transfer = cursor.fetchone()[0]
logger.info(f"转账后余额: {balance_after_transfer}")
# 设置保存点2
cursor.execute("SAVEPOINT sp_after_transfer")
logger.info("设置保存点: sp_after_transfer")
# 操作3: 大额取款（可能失败的业务操作）
cursor.execute(
"UPDATE accounts SET balance = balance - 8000 WHERE id = %s",
(account_id,)
)
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
balance_after_withdrawal = cursor.fetchone()[0]
logger.info(f"大额取款后余额: {balance_after_withdrawal}")
# 检查余额是否为负数
if float(balance_after_withdrawal) < 0:
logger.warning("余额为负数，回滚到保存点 sp_after_transfer")
cursor.execute("ROLLBACK TO SAVEPOINT sp_after_transfer")
# 检查回滚后的余额
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
current_balance = cursor.fetchone()[0]
logger.info(f"回滚后的余额: {current_balance}")
# 尝试较小的取款金额
logger.info("尝试较小的取款金额")
cursor.execute(
"UPDATE accounts SET balance = balance - 3000 WHERE id = %s",
(account_id,)
)
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
final_balance = cursor.fetchone()[0]
logger.info(f"较小取款后余额: {final_balance}")
if float(final_balance) < 0:
logger.error("仍然余额不足，回滚到保存点 sp_after_deposit")
cursor.execute("ROLLBACK TO SAVEPOINT sp_after_deposit")
# 最终余额就是存款后的余额
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
final_balance = cursor.fetchone()[0]
logger.info(f"最终余额: {final_balance}")
# 记录交易历史
cursor.executemany(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
[
("SYSTEM", "ROLLBACK001", 2000.00, "DEPOSIT", "completed"),
("ROLLBACK001", "EXTERNAL", 1500.00, "TRANSFER", "completed"),
("ROLLBACK001", "EXTERNAL", 3000.00, "WITHDRAWAL", "completed")
]
)
# 提交事务
conn.commit()
logger.info("事务提交成功")
# 验证最终状态
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
final_balance = cursor.fetchone()[0]
return {
'initial_balance': initial_balance,
'final_balance': final_balance,
'net_change': float(final_balance) - float(initial_balance)
}
except Exception as e:
conn.rollback()
logger.error(f"回滚保存点演示失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 运行演示
try:
result = rollback_to_savepoint_demo()
print(f"回滚保存点演示结果: {result}")
except Exception as e:
print(f"回滚保存点演示失败: {e}")
---
c.释放保存点
a.功能说明
使用RELEASE SAVEPOINT语句删除指定的保存点，释放相关资源，后续不能再回滚到该保存点。
b.代码示例
---
import pymysql
import logging
logger = logging.getLogger(__name__)
def release_savepoint_demo():
"""释放保存点演示"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
logger.info("=== 释放保存点演示 ===")
# 准备测试数据
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'RELEASE%'")
cursor.execute(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
("RELEASE001", "释放保存点测试", 5000.00)
)
cursor.execute("SELECT id FROM accounts WHERE account_number = %s", ("RELEASE001",))
account_id = cursor.fetchone()[0]
conn.commit()
# 开始事务
conn.begin()
logger.info("开始事务")
# 记录初始余额
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
initial_balance = cursor.fetchone()[0]
# 操作1: 第一次存款
cursor.execute(
"UPDATE accounts SET balance = balance + 1000 WHERE id = %s",
(account_id,)
)
cursor.execute("SAVEPOINT sp_first_deposit")
logger.info("第一次存款并设置保存点 sp_first_deposit")
# 操作2: 第二次存款
cursor.execute(
"UPDATE accounts SET balance = balance + 1500 WHERE id = %s",
(account_id,)
)
cursor.execute("SAVEPOINT sp_second_deposit")
logger.info("第二次存款并设置保存点 sp_second_deposit")
# 操作3: 第三次存款
cursor.execute(
"UPDATE accounts SET balance = balance + 2000 WHERE id = %s",
(account_id,)
)
cursor.execute("SAVEPOINT sp_third_deposit")
logger.info("第三次存款并设置保存点 sp_third_deposit")
# 查询当前余额
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
current_balance = cursor.fetchone()[0]
logger.info(f"所有存款后余额: {current_balance}")
# 释放第二个保存点
cursor.execute("RELEASE SAVEPOINT sp_second_deposit")
logger.info("释放保存点 sp_second_deposit")
# 尝试回滚到已释放的保存点（这会失败）
try:
cursor.execute("ROLLBACK TO SAVEPOINT sp_second_deposit")
logger.info("成功回滚到已释放的保存点（不应该发生）")
except Exception as e:
logger.warning(f"无法回滚到已释放的保存点: {e}")
# 回滚到第一个保存点（仍然有效）
cursor.execute("ROLLBACK TO SAVEPOINT sp_first_deposit")
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
balance_after_rollback = cursor.fetchone()[0]
logger.info(f"回滚到第一个保存点后余额: {balance_after_rollback}")
# 释放第一个保存点
cursor.execute("RELEASE SAVEPOINT sp_first_deposit")
logger.info("释放保存点 sp_first_deposit")
# 再次尝试回滚到第一个保存点（应该失败）
try:
cursor.execute("ROLLBACK TO SAVEPOINT sp_first_deposit")
logger.info("成功回滚到已释放的第一个保存点（不应该发生）")
except Exception as e:
logger.warning(f"无法回滚到已释放的第一个保存点: {e}")
# 执行一些额外操作
cursor.execute(
"UPDATE accounts SET balance = balance + 500 WHERE id = %s",
(account_id,)
)
cursor.execute("SELECT balance FROM accounts WHERE id = %s", (account_id,))
final_balance = cursor.fetchone()[0]
logger.info(f"额外操作后最终余额: {final_balance}")
# 提交事务
conn.commit()
logger.info("事务提交成功")
return {
'initial_balance': initial_balance,
'final_balance': final_balance,
'total_change': float(final_balance) - float(initial_balance)
}
except Exception as e:
conn.rollback()
logger.error(f"释放保存点演示失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 运行演示
try:
result = release_savepoint_demo()
print(f"释放保存点演示结果: {result}")
except Exception as e:
print(f"释放保存点演示失败: {e}")
---
03.复杂业务场景中的保存点应用
a.批量数据处理
a.功能说明
在批量插入或更新数据时，使用保存点实现部分回滚，当某批次操作失败时只回滚该批次，不影响前面已成功的批次。
b.代码示例
---
import pymysql
import logging
import random
logger = logging.getLogger(__name__)
def batch_data_processing_with_savepoints():
"""批量数据处理与保存点应用"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
logger.info("=== 批量数据处理与保存点演示 ===")
# 准备测试数据
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'BATCH%'")
# 生成批量数据
batch_data = []
for i in range(20):
account_number = f"BATCH{str(i+1).zfill(3)}"
account_name = f"批量账户{i+1}"
balance = round(random.uniform(100, 10000), 2)
batch_data.append((account_number, account_name, balance))
logger.info(f"生成 {len(batch_data)} 条账户数据")
# 开始批量处理事务
conn.begin()
logger.info("开始批量处理事务")
processed_count = 0
failed_batches = []
batch_size = 5 # 每批处理5条记录
for batch_num in range(0, len(batch_data), batch_size):
batch = batch_data[batch_num:batch_num + batch_size]
batch_start = batch_num + 1
batch_end = min(batch_num + batch_size, len(batch_data))
logger.info(f"处理第 {batch_start}-{batch_end} 条记录")
# 设置批次保存点
savepoint_name = f"batch_{batch_start}_{batch_end}"
cursor.execute(f"SAVEPOINT {savepoint_name}")
logger.info(f"设置保存点: {savepoint_name}")
batch_success = True
batch_results = []
try:
# 处理当前批次
for account_number, account_name, balance in batch:
try:
# 插入账户
cursor.execute(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
(account_number, account_name, balance)
)
account_id = cursor.lastrowid
# 模拟一些业务处理（比如计算利息）
interest = balance * 0.001 # 0.1% 利息
if balance > 5000: # 大额账户给更高利息
interest = balance * 0.002 # 0.2% 利息
# 更新余额（包含利息）
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(interest, account_id)
)
batch_results.append({
'account_number': account_number,
'original_balance': balance,
'interest': interest,
'final_balance': balance + interest
})
# 模拟可能的错误（10%概率）
if random.random() < 0.1:
raise Exception(f"随机处理错误: {account_number}")
except Exception as e:
logger.error(f"处理账户 {account_number} 失败: {e}")
batch_success = False
break
if batch_success:
# 批次处理成功，记录交易
for result in batch_results:
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
("SYSTEM", result['account_number'], result['interest'],
"INTEREST", "completed")
)
processed_count += len(batch)
logger.info(f"第 {batch_start}-{batch_end} 条记录处理成功")
# 释放成功批次的保存点
cursor.execute(f"RELEASE SAVEPOINT {savepoint_name}")
logger.info(f"释放保存点: {savepoint_name}")
else:
# 批次处理失败，回滚到批次开始点
cursor.execute(f"ROLLBACK TO SAVEPOINT {savepoint_name}")
logger.warning(f"第 {batch_start}-{batch_end} 条记录处理失败，已回滚")
failed_batches.append((batch_start, batch_end))
except Exception as e:
# 批次处理异常，回滚到批次开始点
try:
cursor.execute(f"ROLLBACK TO SAVEPOINT {savepoint_name}")
logger.warning(f"第 {batch_start}-{batch_end} 条记录异常，已回滚: {e}")
except Exception as rollback_error:
logger.error(f"回滚保存点失败: {rollback_error}")
failed_batches.append((batch_start, batch_end))
# 查询最终结果
cursor.execute(
"SELECT COUNT(*), SUM(balance) FROM accounts WHERE account_number LIKE 'BATCH%'"
)
total_accounts, total_balance = cursor.fetchone()
cursor.execute(
"SELECT COUNT(*) FROM transactions WHERE transaction_type = 'INTEREST' AND status = 'completed'"
)
interest_transactions = cursor.fetchone()[0]
logger.info(f"批量处理完成: 成功 {processed_count}/{len(batch_data)} 条")
logger.info(f"失败批次: {failed_batches}")
logger.info(f"总账户数: {total_accounts}, 总余额: {total_balance}")
logger.info(f"利息交易数: {interest_transactions}")
# 提交事务
conn.commit()
logger.info("批量处理事务提交成功")
return {
'total_records': len(batch_data),
'processed_count': processed_count,
'failed_batches': failed_batches,
'total_accounts': total_accounts,
'total_balance': total_balance,
'success_rate': processed_count / len(batch_data) if batch_data else 0
}
except Exception as e:
conn.rollback()
logger.error(f"批量数据处理失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 运行演示
try:
result = batch_data_processing_with_savepoints()
print(f"批量数据处理结果: {result}")
except Exception as e:
print(f"批量数据处理失败: {e}")
---
b.复杂交易流程
a.功能说明
在涉及多个步骤的复杂交易流程中使用保存点，实现精细化的错误控制和回滚策略，提高系统的容错能力。
b.代码示例
---
import pymysql
import logging
import time
logger = logging.getLogger(__name__)
def complex_transaction_workflow():
"""复杂交易流程与保存点应用"""
conn = pymysql.Connect(
host='localhost',
user='root',
password='password',
database='bank_db',
charset='utf8mb4',
autocommit=False
)
cursor = conn.cursor()
try:
logger.info("=== 复杂交易流程与保存点演示 ===")
# 准备测试账户
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'COMPLEX%'")
test_accounts = [
("COMPLEX001", "复杂交易账户1", 10000.00),
("COMPLEX002", "复杂交易账户2", 8000.00),
("COMPLEX003", "复杂交易账户3", 15000.00)
]
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
test_accounts
)
conn.commit()
# 开始复杂交易流程
conn.begin()
logger.info("开始复杂交易流程")
# 步骤1: 账户验证和锁定
logger.info("步骤1: 账户验证和锁定")
cursor.execute("SAVEPOINT sp_validation")
account_balances = {}
for account_number, _, _ in test_accounts:
cursor.execute(
"SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(account_number,)
)
result = cursor.fetchone()
if not result:
raise Exception(f"账户 {account_number} 不存在")
account_balances[account_number] = {
'id': result[0],
'balance': float(result[1])
}
logger.info(f"账户验证完成: {account_balances}")
# 步骤2: 预授权检查
logger.info("步骤2: 预授权检查")
cursor.execute("SAVEPOINT sp_preauth")
# 模拟预授权检查（验证交易权限）
authorization_required = 5000.00 # 超过5000需要授权
total_operation_amount = 12000.00
if total_operation_amount > authorization_required:
# 模拟授权检查
logger.info(f"交易金额 {total_operation_amount} 超过授权限额 {authorization_required}")
# 模拟授权失败（30%概率）
import random
if random.random() < 0.3:
logger.warning("预授权检查失败")
cursor.execute("ROLLBACK TO SAVEPOINT sp_validation")
raise Exception("交易未获得授权")
else:
logger.info("预授权检查通过")
# 步骤3: 资金准备阶段
logger.info("步骤3: 资金准备阶段")
cursor.execute("SAVEPOINT sp_funding")
try:
# 从COMPLEX001转出5000到临时账户
transfer_amount = 5000.00
from_account = "COMPLEX001"
to_account = "TEMP_ACCOUNT"
if account_balances[from_account]['balance'] < transfer_amount:
raise Exception(f"账户 {from_account} 余额不足")
# 记录资金转移
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status)
VALUES (%s, %s, %s, %s, %s)""",
(from_account, to_account, transfer_amount, "TRANSFER", "processing")
)
transaction_id = cursor.lastrowid
# 更新余额
cursor.execute(
"UPDATE accounts SET balance = balance - %s WHERE id = %s",
(transfer_amount, account_balances[from_account]['id'])
)
account_balances[from_account]['balance'] -= transfer_amount
logger.info(f"资金准备完成: {from_account} 转出 {transfer_amount}")
except Exception as e:
logger.error(f"资金准备失败: {e}")
cursor.execute("ROLLBACK TO SAVEPOINT sp_preauth")
raise
# 步骤4: 主要业务操作
logger.info("步骤4: 主要业务操作")
cursor.execute("SAVEPOINT sp_main_operation")
try:
# 执行主要业务逻辑（比如投资操作）
investment_amount = 3000.00
# 更新COMPLEX002作为投资接收方
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(investment_amount, account_balances["COMPLEX002"]['id'])
)
# 记录投资交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
("TEMP_ACCOUNT", "COMPLEX002", investment_amount, "INVESTMENT", "completed")
)
account_balances["COMPLEX002"]['balance'] += investment_amount
remaining_temp = transfer_amount - investment_amount
logger.info(f"主要业务操作完成: 投资 {investment_amount}")
# 步骤5: 清算和结算
logger.info("步骤5: 清算和结算")
cursor.execute("SAVEPOINT sp_settlement")
# 将剩余资金转回COMPLEX001
if remaining_temp > 0:
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(remaining_temp, account_balances["COMPLEX001"]['id'])
)
# 记录退款交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
("TEMP_ACCOUNT", "COMPLEX001", remaining_temp, "REFUND", "completed")
)
account_balances["COMPLEX001"]['balance'] += remaining_temp
logger.info(f"资金退回: {remaining_temp}")
# 更新交易状态
cursor.execute(
"UPDATE transactions SET status = 'completed', processed_at = CURRENT_TIMESTAMP WHERE id = %s",
(transaction_id,)
)
# 给COMPLEX003发放手续费
fee = 100.00
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(fee, account_balances["COMPLEX003"]['id'])
)
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
("SYSTEM", "COMPLEX003", fee, "FEE", "completed")
)
logger.info(f"手续费发放: {fee}")
except Exception as e:
logger.error(f"主要业务操作失败: {e}")
cursor.execute("ROLLBACK TO SAVEPOINT sp_funding")
logger.info("回滚到资金准备阶段，已恢复资金")
raise
# 最终验证
logger.info("最终验证:")
for account_number, _, initial_balance in test_accounts:
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = %s",
(account_number,)
)
current_balance = cursor.fetchone()[0]
change = float(current_balance) - initial_balance
logger.info(f"{account_number}: {initial_balance} -> {current_balance} (变化: {change:+.2f})")
# 提交整个复杂交易
conn.commit()
logger.info("复杂交易流程提交成功")
return {
'status': 'success',
'message': '复杂交易流程完成',
'processed_accounts': len(test_accounts)
}
except Exception as e:
conn.rollback()
logger.error(f"复杂交易流程失败: {e}")
return {
'status': 'failed',
'message': str(e)
}
finally:
cursor.close()
conn.close()
# 运行演示
try:
result = complex_transaction_workflow()
print(f"复杂交易流程结果: {result}")
except Exception as e:
print(f"复杂交易流程失败: {e}")
---
3.4 死锁处理
01.死锁基本概念
a.死锁定义
a.功能说明
死锁是指两个或多个事务因争夺资源而造成的一种互相等待的现象，若无外力作用，它们都将无法继续执行。
b.死锁产生的必要条件
互斥条件：资源不能被多个事务同时使用
占有且等待：一个事务已占有资源，同时等待其他事务释放资源
不可剥夺条件：资源不能被强制从持有它的事务中剥夺
循环等待：存在事务等待链，形成循环等待关系
b.MySQL死锁机制
a.功能说明
MySQL使用InnoDB存储引擎自动检测死锁，并选择一个代价最小的事务进行回滚，打破死锁循环。
b.死锁检测策略
自动死锁检测：InnoDB会主动检测等待图中是否存在环
超时机制：当事务等待时间超过innodb_lock_wait_timeout时自动回滚
死锁回滚：选择撤销行数最少的事务
02.死锁检测与分析
a.死锁检测方法
a.功能说明
通过MySQL的错误信息、日志记录和系统表来检测和分析死锁情况，理解死锁发生的原因和影响范围。
b.代码示例
---
import pymysql
import logging
import threading
import time
import random
logger = logging.getLogger(__name__)
class DeadlockDetector:
"""死锁检测器"""
def __init__(self, connection_config):
self.connection_config = connection_config
self.deadlock_history = []
self.lock = threading.Lock()
def check_recent_deadlocks(self, minutes=5):
"""检查最近的死锁记录"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor(pymysql.cursors.DictCursor)
try:
# 查询最近的错误日志
cursor.execute("""
SHOW ENGINE INNODB STATUS
""")
result = cursor.fetchone()
# 解析死锁信息
deadlock_info = self._parse_deadlock_info(result['Status'])
return deadlock_info
except Exception as e:
logger.error(f"检查死锁失败: {e}")
return []
finally:
cursor.close()
conn.close()
def _parse_deadlock_info(self, status_text):
"""解析InnoDB状态中的死锁信息"""
deadlock_info = []
# 查找死锁相关的段落
lines = status_text.split('\n')
in_deadlock_section = False
current_deadlock = {}
for line in lines:
if 'LATEST DETECTED DEADLOCK' in line:
in_deadlock_section = True
continue
if in_deadlock_section:
if 'WE ROLL BACK TRANSACTION' in line:
if current_deadlock:
deadlock_info.append(current_deadlock)
current_deadlock = {}
in_deadlock_section = False
elif 'TRANSACTION' in line:
if current_deadlock:
deadlock_info.append(current_deadlock)
current_deadlock = {'transaction': line.strip()}
elif 'MySQL thread id' in line:
current_deadlock['thread_id'] = line.strip()
elif '*** (1) WAITING FOR THIS LOCK TO BE GRANTED:' in line:
current_deadlock['waiting_lock'] = True
elif '*** (2) HOLDS THE LOCK(S):' in line:
current_deadlock['holding_lock'] = True
if current_deadlock:
deadlock_info.append(current_deadlock)
return deadlock_info
def simulate_deadlock_scenario(self):
"""模拟死锁场景"""
logger.info("=== 开始模拟死锁场景 ===")
def transaction1():
"""事务1：先锁定account1，然后锁定account2"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
conn.begin()
logger.info("事务1开始")
# 锁定第一个账户
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = 'DEADLOCK001' FOR UPDATE"
)
balance1 = cursor.fetchone()
logger.info(f"事务1锁定DEADLOCK001，余额: {balance1}")
# 等待一段时间，增加死锁概率
time.sleep(1)
# 尝试锁定第二个账户（这里可能发生死锁）
try:
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = 'DEADLOCK002' FOR UPDATE"
)
balance2 = cursor.fetchone()
logger.info(f"事务1锁定DEADLOCK002，余额: {balance2}")
# 更新操作
cursor.execute(
"UPDATE accounts SET balance = balance - 100 WHERE account_number = 'DEADLOCK001'"
)
cursor.execute(
"UPDATE accounts SET balance = balance + 100 WHERE account_number = 'DEADLOCK002'"
)
conn.commit()
logger.info("事务1提交成功")
except pymysql.err.OperationalError as e:
if 'deadlock' in str(e).lower():
logger.warning(f"事务1遇到死锁: {e}")
conn.rollback()
return {'status': 'deadlock', 'message': str(e)}
else:
raise e
except Exception as e:
conn.rollback()
logger.error(f"事务1失败: {e}")
return {'status': 'error', 'message': str(e)}
finally:
cursor.close()
conn.close()
def transaction2():
"""事务2：先锁定account2，然后锁定account1"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
conn.begin()
logger.info("事务2开始")
# 锁定第二个账户
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = 'DEADLOCK002' FOR UPDATE"
)
balance2 = cursor.fetchone()
logger.info(f"事务2锁定DEADLOCK002，余额: {balance2}")
# 等待一段时间，增加死锁概率
time.sleep(0.5)
# 尝试锁定第一个账户（这里可能发生死锁）
try:
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = 'DEADLOCK001' FOR UPDATE"
)
balance1 = cursor.fetchone()
logger.info(f"事务2锁定DEADLOCK001，余额: {balance1}")
# 更新操作
cursor.execute(
"UPDATE accounts SET balance = balance - 200 WHERE account_number = 'DEADLOCK002'"
)
cursor.execute(
"UPDATE accounts SET balance = balance + 200 WHERE account_number = 'DEADLOCK001'"
)
conn.commit()
logger.info("事务2提交成功")
except pymysql.err.OperationalError as e:
if 'deadlock' in str(e).lower():
logger.warning(f"事务2遇到死锁: {e}")
conn.rollback()
return {'status': 'deadlock', 'message': str(e)}
else:
raise e
except Exception as e:
conn.rollback()
logger.error(f"事务2失败: {e}")
return {'status': 'error', 'message': str(e)}
finally:
cursor.close()
conn.close()
# 准备测试数据
self._prepare_deadlock_test_data()
# 启动两个事务，制造死锁
t1 = threading.Thread(target=transaction1)
t2 = threading.Thread(target=transaction2)
t1.start()
time.sleep(0.2) # 稍微错开启动时间
t2.start()
# 等待完成
t1.join()
t2.join()
# 检查死锁记录
deadlocks = self.check_recent_deadlocks()
logger.info(f"检测到的死锁数量: {len(deadlocks)}")
return deadlocks
def _prepare_deadlock_test_data(self):
"""准备死锁测试数据"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
# 删除旧的测试数据
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'DEADLOCK%'")
# 插入测试账户
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[
("DEADLOCK001", "死锁测试账户1", 5000.00),
("DEADLOCK002", "死锁测试账户2", 3000.00)
]
)
conn.commit()
logger.info("死锁测试数据准备完成")
except Exception as e:
conn.rollback()
logger.error(f"准备死锁测试数据失败: {e}")
raise
finally:
cursor.close()
conn.close()
# 使用示例
def demonstrate_deadlock_detection():
"""演示死锁检测"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4',
'autocommit': False
}
detector = DeadlockDetector(connection_config)
try:
# 运行多次死锁模拟，增加检测概率
for i in range(3):
logger.info(f"第 {i+1} 次死锁模拟")
deadlocks = detector.simulate_deadlock_scenario()
if deadlocks:
logger.info(f"第 {i+1} 次检测到死锁: {len(deadlocks)}")
for deadlock in deadlocks:
logger.info(f"死锁详情: {deadlock}")
else:
logger.info(f"第 {i+1} 次未检测到死锁")
time.sleep(2) # 等待一段时间再进行下一次
except Exception as e:
logger.error(f"死锁检测演示失败: {e}")
# 运行演示
demonstrate_deadlock_detection()
---
b.死锁分析工具
a.功能说明
使用系统表和性能模式来分析死锁的详细信息，包括事务ID、锁类型、等待时间、SQL语句等。
b.代码示例
---
import pymysql
import logging
import json
from datetime import datetime, timedelta
logger = logging.getLogger(__name__)
class DeadlockAnalyzer:
"""死锁分析器"""
def __init__(self, connection_config):
self.connection_config = connection_config
def analyze_current_locks(self):
"""分析当前的锁情况"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor(pymysql.cursors.DictCursor)
try:
# 查询当前锁等待情况
cursor.execute("""
SELECT
r.trx_id AS waiting_trx_id,
r.trx_mysql_thread_id AS waiting_thread,
r.trx_query AS waiting_query,
b.trx_id AS blocking_trx_id,
b.trx_mysql_thread_id AS blocking_thread,
b.trx_query AS blocking_query,
l.lock_table,
l.lock_index,
l.lock_mode
FROM information_schema.innodb_lock_waits w
INNER JOIN information_schema.innodb_trx b ON b.trx_id = w.blocking_trx_id
INNER JOIN information_schema.innodb_trx r ON r.trx_id = w.requesting_trx_id
INNER JOIN information_schema.innodb_locks l ON l.lock_trx_id = w.blocking_trx_id
ORDER BY r.trx_id
""")
lock_waits = cursor.fetchall()
# 查询当前活跃事务
cursor.execute("""
SELECT
trx_id,
trx_state,
trx_started,
trx_mysql_thread_id,
trx_query,
trx_rows_locked,
trx_rows_modified
FROM information_schema.innodb_trx
WHERE trx_state != 'RUNNING'
ORDER BY trx_started
""")
active_transactions = cursor.fetchall()
return {
'lock_waits': lock_waits,
'active_transactions': active_transactions,
'analysis_time': datetime.now().isoformat()
}
except Exception as e:
logger.error(f"分析当前锁情况失败: {e}")
return {'error': str(e)}
finally:
cursor.close()
conn.close()
def get_transaction_locks(self, thread_id=None):
"""获取指定事务的锁信息"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor(pymysql.cursors.DictCursor)
try:
if thread_id:
# 查询指定线程的锁
cursor.execute("""
SELECT
l.lock_trx_id,
l.lock_mode,
l.lock_type,
l.lock_table,
l.lock_index,
l.lock_space,
l.lock_page,
l.lock_rec,
l.lock_data,
p.id AS process_id,
p.user,
p.host,
p.db,
p.command,
p.time,
p.state,
p.info
FROM information_schema.innodb_locks l
LEFT JOIN information_schema.processlist p ON l.lock_trx_id = p.id
WHERE p.id = %s
ORDER BY l.lock_trx_id
""", (thread_id,))
else:
# 查询所有锁
cursor.execute("""
SELECT
l.lock_trx_id,
l.lock_mode,
l.lock_type,
l.lock_table,
l.lock_index,
l.lock_space,
l.lock_page,
l.lock_rec,
l.lock_data,
p.id AS process_id,
p.user,
p.host,
p.db,
p.command,
p.time,
p.state,
p.info
FROM information_schema.innodb_locks l
LEFT JOIN information_schema.processlist p ON l.lock_trx_id = p.id
ORDER BY l.lock_trx_id
""")
locks = cursor.fetchall()
# 格式化结果
formatted_locks = []
for lock in locks:
formatted_lock = {
'transaction_id': lock['lock_trx_id'],
'lock_mode': lock['lock_mode'],
'lock_type': lock['lock_type'],
'table': lock['lock_table'],
'index': lock['lock_index'],
'lock_data': lock['lock_data'],
'process_info': {
'id': lock['process_id'],
'user': lock['user'],
'host': lock['host'],
'database': lock['db'],
'command': lock['command'],
'duration': lock['time'],
'state': lock['state'],
'query': lock['info']
}
}
formatted_locks.append(formatted_lock)
return formatted_locks
except Exception as e:
logger.error(f"获取事务锁信息失败: {e}")
return []
finally:
cursor.close()
conn.close()
def detect_potential_deadlocks(self):
"""检测潜在的死锁风险"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor(pymysql.cursors.DictCursor)
try:
# 查询长时间运行的事务
cursor.execute("""
SELECT
trx_id,
trx_mysql_thread_id,
trx_started,
TIMESTAMPDIFF(SECOND, trx_started, NOW()) as duration_seconds,
trx_rows_locked,
trx_rows_modified
FROM information_schema.innodb_trx
WHERE TIMESTAMPDIFF(SECOND, trx_started, NOW()) > 30
ORDER BY duration_seconds DESC
""")
long_running_transactions = cursor.fetchall()
# 分析锁等待链
cursor.execute("""
SELECT
r.trx_id AS waiting_trx,
r.trx_mysql_thread_id AS waiting_thread,
b.trx_id AS blocking_trx,
b.trx_mysql_thread_id AS blocking_thread,
l.lock_table,
l.lock_mode
FROM information_schema.innodb_lock_waits w
INNER JOIN information_schema.innodb_trx r ON r.trx_id = w.requesting_trx_id
INNER JOIN information_schema.innodb_trx b ON b.trx_id = w.blocking_trx_id
INNER JOIN information_schema.innodb_locks l ON l.lock_trx_id = w.blocking_trx_id
""")
lock_chains = cursor.fetchall()
# 检测循环等待
deadlock_risks = []
for chain in lock_chains:
# 检查是否存在反向等待
reverse_exists = any(
c['waiting_trx'] == chain['blocking_trx'] and
c['blocking_trx'] == chain['waiting_trx']
for c in lock_chains
)
if reverse_exists:
deadlock_risks.append({
'risk_type': 'circular_wait',
'waiting_thread': chain['waiting_thread'],
'blocking_thread': chain['blocking_thread'],
'table': chain['lock_table'],
'lock_mode': chain['lock_mode'],
'severity': 'high'
})
# 检测长时间锁等待
for chain in lock_chains:
waiting_thread = chain['waiting_thread']
cursor.execute("""
SELECT time FROM information_schema.processlist WHERE id = %s
""", (waiting_thread,))
result = cursor.fetchone()
if result and result['time'] > 10: # 等待超过10秒
deadlock_risks.append({
'risk_type': 'long_wait',
'waiting_thread': waiting_thread,
'wait_duration': result['time'],
'table': chain['lock_table'],
'severity': 'medium'
})
return {
'long_running_transactions': long_running_transactions,
'lock_chains': lock_chains,
'deadlock_risks': deadlock_risks,
'analysis_time': datetime.now().isoformat()
}
except Exception as e:
logger.error(f"检测潜在死锁风险失败: {e}")
return {'error': str(e)}
finally:
cursor.close()
conn.close()
def generate_deadlock_report(self):
"""生成死锁分析报告"""
logger.info("开始生成死锁分析报告")
# 分析当前锁情况
current_locks = self.analyze_current_locks()
# 检测潜在死锁风险
risks = self.detect_potential_deadlocks()
# 获取所有锁信息
all_locks = self.get_transaction_locks()
report = {
'report_time': datetime.now().isoformat(),
'summary': {
'total_lock_waits': len(current_locks.get('lock_waits', [])),
'active_transactions': len(current_locks.get('active_transactions', [])),
'total_locks': len(all_locks),
'deadlock_risks': len(risks.get('deadlock_risks', []))
},
'current_locks': current_locks,
'potential_risks': risks,
'all_locks': all_locks,
'recommendations': self._generate_recommendations(risks)
}
return report
def _generate_recommendations(self, risks):
"""生成优化建议"""
recommendations = []
if not risks or 'deadlock_risks' not in risks:
return recommendations
deadlock_risks = risks['deadlock_risks']
# 分析风险类型
risk_types = [risk['risk_type'] for risk in deadlock_risks]
risk_count = len(deadlock_risks)
if 'circular_wait' in risk_types:
recommendations.append({
'priority': 'high',
'category': 'deadlock_prevention',
'issue': '检测到循环等待',
'recommendation': '重新设计事务逻辑，确保所有事务按相同顺序访问资源',
'action': '检查应用程序代码，统一资源访问顺序'
})
if 'long_wait' in risk_types:
recommendations.append({
'priority': 'medium',
'category': 'performance',
'issue': '检测到长时间锁等待',
'recommendation': '优化事务处理逻辑，减少锁持有时间',
'action': '分解大事务为小事务，避免长时间锁定资源'
})
if risk_count > 5:
recommendations.append({
'priority': 'high',
'category': 'system_health',
'issue': f'检测到{risk_count}个死锁风险',
'recommendation': '系统存在大量并发冲突，需要整体优化',
'action': '考虑使用乐观锁、读写分离或缓存策略'
})
return recommendations
# 使用示例
def demonstrate_deadlock_analysis():
"""演示死锁分析"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4'
}
analyzer = DeadlockAnalyzer(connection_config)
try:
# 生成完整的死锁分析报告
report = analyzer.generate_deadlock_report()
print("=== 死锁分析报告 ===")
print(json.dumps(report, indent=2, ensure_ascii=False))
# 输出建议
if report['recommendations']:
print("\n=== 优化建议 ===")
for rec in report['recommendations']:
print(f"[{rec['priority'].upper()}] {rec['category']}")
print(f"问题: {rec['issue']}")
print(f"建议: {rec['recommendation']}")
print(f"行动: {rec['action']}\n")
except Exception as e:
logger.error(f"死锁分析演示失败: {e}")
# 运行演示
demonstrate_deadlock_analysis()
---
03.死锁预防策略
a.资源访问顺序统一
a.功能说明
通过确保所有事务按相同的顺序访问资源，从根本上避免循环等待条件的发生。
b.代码示例
---
import pymysql
import logging
import threading
import time
logger = logging.getLogger(__name__)
class ResourceManager:
"""资源管理器 - 统一资源访问顺序"""
def __init__(self, connection_config):
self.connection_config = connection_config
self.resource_order = {} # 资源访问顺序映射
def define_resource_order(self, resources):
"""定义资源访问顺序"""
# 按照某种规则（如账户号排序）定义访问顺序
sorted_resources = sorted(resources)
for i, resource in enumerate(sorted_resources):
self.resource_order[resource] = i
logger.info(f"定义资源访问顺序: {self.resource_order}")
def get_ordered_resources(self, resources):
"""获取按顺序排列的资源列表"""
return sorted(resources, key=lambda x: self.resource_order.get(x, float('inf')))
def execute_with_ordered_locks(self, operations):
"""按顺序执行需要加锁的操作"""
# 收集所有需要锁定的资源
all_resources = set()
for op in operations:
if 'resources' in op:
all_resources.update(op['resources'])
# 定义资源访问顺序
self.define_resource_order(all_resources)
# 按顺序执行操作
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
conn.begin()
logger.info("开始有序事务执行")
# 按预定顺序锁定所有资源
ordered_resources = self.get_ordered_resources(all_resources)
locked_resources = {}
for resource in ordered_resources:
# 锁定资源
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = %s FOR UPDATE",
(resource,)
)
result = cursor.fetchone()
locked_resources[resource] = result[0] if result else 0
logger.info(f"锁定资源: {resource}")
# 执行所有操作
for op in operations:
op_type = op['type']
if op_type == 'transfer':
from_account = op['from_account']
to_account = op['to_account']
amount = op['amount']
# 验证余额
if locked_resources.get(from_account, 0) >= amount:
# 执行转账
cursor.execute(
"UPDATE accounts SET balance = balance - %s WHERE account_number = %s",
(amount, from_account)
)
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE account_number = %s",
(amount, to_account)
)
# 记录交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
(from_account, to_account, amount, 'TRANSFER', 'completed')
)
# 更新锁定资源缓存
locked_resources[from_account] -= amount
locked_resources[to_account] = locked_resources.get(to_account, 0) + amount
logger.info(f"转账完成: {from_account} -> {to_account}, 金额: {amount}")
else:
raise Exception(f"账户 {from_account} 余额不足")
elif op_type == 'deposit':
account = op['account']
amount = op['amount']
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE account_number = %s",
(amount, account)
)
# 记录存款
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
('SYSTEM', account, amount, 'DEPOSIT', 'completed')
)
locked_resources[account] += amount
logger.info(f"存款完成: {account}, 金额: {amount}")
conn.commit()
logger.info("有序事务执行完成")
return True
except Exception as e:
conn.rollback()
logger.error(f"有序事务执行失败: {e}")
return False
finally:
cursor.close()
conn.close()
def demonstrate_ordered_access():
"""演示有序资源访问防止死锁"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4',
'autocommit': False
}
# 准备测试数据
conn = pymysql.Connect(**connection_config)
cursor = conn.cursor()
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'ORDER%'")
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[
("ORDER001", "有序访问账户1", 10000.00),
("ORDER002", "有序访问账户2", 8000.00),
("ORDER003", "有序访问账户3", 5000.00)
]
)
conn.commit()
cursor.close()
conn.close()
resource_manager = ResourceManager(connection_config)
def worker_thread(thread_id, operations):
"""工作线程"""
logger.info(f"线程 {thread_id} 开始执行")
success = resource_manager.execute_with_ordered_locks(operations)
logger.info(f"线程 {thread_id} 执行结果: {'成功' if success else '失败'}")
return success
# 定义两组可能产生死锁的操作
operations1 = [
{
'type': 'transfer',
'from_account': 'ORDER001',
'to_account': 'ORDER002',
'amount': 1000.00,
'resources': ['ORDER001', 'ORDER002']
},
{
'type': 'transfer',
'from_account': 'ORDER002',
'to_account': 'ORDER003',
'amount': 500.00,
'resources': ['ORDER002', 'ORDER003']
}
]
operations2 = [
{
'type': 'transfer',
'from_account': 'ORDER003',
'to_account': 'ORDER001',
'amount': 800.00,
'resources': ['ORDER003', 'ORDER001']
},
{
'type': 'deposit',
'account': 'ORDER002',
'amount': 200.00,
'resources': ['ORDER002']
}
]
# 启动两个线程，如果访问顺序不当会产生死锁
threads = []
for i in range(3): # 运行多次测试
t1 = threading.Thread(target=worker_thread, args=(f"1-{i}", operations1))
t2 = threading.Thread(target=worker_thread, args=(f"2-{i}", operations2))
threads.extend([t1, t2])
t1.start()
time.sleep(0.1) # 稍微错开启动时间
t2.start()
t1.join()
t2.join()
logger.info(f"第 {i+1} 轮测试完成")
logger.info("所有测试完成，未发生死锁")
# 运行演示
demonstrate_ordered_access()
---
b.事务粒度控制
a.功能说明
通过控制事务的粒度和持续时间，减少锁的持有时间和冲突概率，从而降低死锁发生的可能性。
b.代码示例
---
import pymysql
import logging
import threading
import time
from contextlib import contextmanager
logger = logging.getLogger(__name__)
class TransactionGranularityManager:
"""事务粒度管理器"""
def __init__(self, connection_config):
self.connection_config = connection_config
@contextmanager
def short_transaction(self):
"""短事务上下文管理器"""
conn = pymysql.Connect(**self.connection_config)
try:
yield conn
except Exception as e:
conn.rollback()
logger.error(f"短事务执行失败: {e}")
raise
finally:
conn.close()
def execute_small_transactions(self, operations):
"""将大操作分解为多个小事务"""
successful_operations = []
failed_operations = []
for i, operation in enumerate(operations):
logger.info(f"执行小事务 {i+1}/{len(operations)}")
try:
with self.short_transaction() as conn:
cursor = conn.cursor()
conn.begin()
# 执行单个操作
if operation['type'] == 'transfer':
result = self._execute_single_transfer(cursor, operation)
elif operation['type'] == 'deposit':
result = self._execute_single_deposit(cursor, operation)
else:
raise Exception(f"不支持的操作类型: {operation['type']}")
conn.commit()
successful_operations.append({
'operation': operation,
'result': result,
'transaction_time': time.time()
})
logger.info(f"小事务 {i+1} 执行成功")
except Exception as e:
failed_operations.append({
'operation': operation,
'error': str(e),
'transaction_time': time.time()
})
logger.warning(f"小事务 {i+1} 执行失败: {e}")
return {
'successful_count': len(successful_operations),
'failed_count': len(failed_operations),
'successful_operations': successful_operations,
'failed_operations': failed_operations
}
def _execute_single_transfer(self, cursor, operation):
"""执行单个转账操作"""
from_account = operation['from_account']
to_account = operation['to_account']
amount = operation['amount']
# 检查转出账户余额
cursor.execute(
"SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(from_account,)
)
from_result = cursor.fetchone()
if not from_result:
raise Exception(f"转出账户 {from_account} 不存在")
from_id, from_balance = from_result
if float(from_balance) < amount:
raise Exception(f"账户 {from_account} 余额不足")
# 检查转入账户
cursor.execute(
"SELECT id FROM accounts WHERE account_number = %s",
(to_account,)
)
to_result = cursor.fetchone()
if not to_result:
raise Exception(f"转入账户 {to_account} 不存在")
to_id = to_result[0]
# 执行转账
cursor.execute(
"UPDATE accounts SET balance = balance - %s WHERE id = %s",
(amount, from_id)
)
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(amount, to_id)
)
# 记录交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
(from_account, to_account, amount, 'TRANSFER', 'completed')
)
return {
'from_account': from_account,
'to_account': to_account,
'amount': amount,
'new_from_balance': float(from_balance) - amount,
'transaction_id': cursor.lastrowid
}
def _execute_single_deposit(self, cursor, operation):
"""执行单个存款操作"""
account = operation['account']
amount = operation['amount']
# 检查账户
cursor.execute(
"SELECT id FROM accounts WHERE account_number = %s",
(account,)
)
result = cursor.fetchone()
if not result:
raise Exception(f"账户 {account} 不存在")
account_id = result[0]
# 执行存款
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(amount, account_id)
)
# 记录交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
('SYSTEM', account, amount, 'DEPOSIT', 'completed')
)
return {
'account': account,
'amount': amount,
'transaction_id': cursor.lastrowid
}
def batch_process_with_small_transactions(self, batch_data):
"""使用小事务批量处理数据"""
logger.info(f"开始批量处理 {len(batch_data)} 条记录")
# 将操作分组，每个组作为一个小事务
operations = []
for data in batch_data:
if data['operation'] == 'transfer':
operations.append({
'type': 'transfer',
'from_account': data['from_account'],
'to_account': data['to_account'],
'amount': data['amount']
})
elif data['operation'] == 'deposit':
operations.append({
'type': 'deposit',
'account': data['account'],
'amount': data['amount']
})
# 并行执行小事务（适度并行）
max_workers = 3
chunks = [operations[i:i + max_workers] for i in range(0, len(operations), max_workers)]
all_results = []
for chunk_idx, chunk in enumerate(chunks):
logger.info(f"执行批次 {chunk_idx + 1}/{len(chunks)}")
# 并行执行当前批次
threads = []
results = []
def execute_operation(op):
return self.execute_small_transactions([op])
for op in chunk:
thread = threading.Thread(
target=lambda: results.append(execute_operation(op))
)
threads.append(thread)
thread.start()
# 等待当前批次完成
for thread in threads:
thread.join()
all_results.extend(results)
# 汇总结果
total_successful = sum(r['successful_count'] for r in all_results)
total_failed = sum(r['failed_count'] for r in all_results)
return {
'total_operations': len(operations),
'successful_count': total_successful,
'failed_count': total_failed,
'success_rate': total_successful / len(operations) if operations else 0,
'details': all_results
}
def demonstrate_granularity_control():
"""演示事务粒度控制"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4',
'autocommit': False
}
# 准备测试数据
conn = pymysql.Connect(**connection_config)
cursor = conn.cursor()
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'GRANULAR%'")
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[
("GRANULAR001", "粒度控制账户1", 5000.00),
("GRANULAR002", "粒度控制账户2", 3000.00),
("GRANULAR003", "粒度控制账户3", 2000.00),
("GRANULAR004", "粒度控制账户4", 1000.00)
]
)
conn.commit()
cursor.close()
conn.close()
granularity_manager = TransactionGranularityManager(connection_config)
# 准备批量操作数据
batch_operations = [
{'operation': 'transfer', 'from_account': 'GRANULAR001', 'to_account': 'GRANULAR002', 'amount': 500.00},
{'operation': 'deposit', 'account': 'GRANULAR003', 'amount': 200.00},
{'operation': 'transfer', 'from_account': 'GRANULAR002', 'to_account': 'GRANULAR004', 'amount': 300.00},
{'operation': 'transfer', 'from_account': 'GRANULAR004', 'to_account': 'GRANULAR001', 'amount': 100.00},
{'operation': 'deposit', 'account': 'GRANULAR003', 'amount': 150.00},
{'operation': 'transfer', 'from_account': 'GRANULAR001', 'to_account': 'GRANULAR003', 'amount': 250.00}
]
try:
# 使用小事务批量处理
result = granularity_manager.batch_process_with_small_transactions(batch_operations)
print("=== 事务粒度控制结果 ===")
print(f"总操作数: {result['total_operations']}")
print(f"成功操作数: {result['successful_count']}")
print(f"失败操作数: {result['failed_count']}")
print(f"成功率: {result['success_rate']:.2%}")
if result['failed_count'] > 0:
print("\n失败操作详情:")
for detail in result['details']:
if detail['failed_operations']:
for failed_op in detail['failed_operations']:
print(f"- {failed_op['operation']}: {failed_op['error']}")
logger.info("事务粒度控制演示完成")
except Exception as e:
logger.error(f"事务粒度控制演示失败: {e}")
# 运行演示
demonstrate_granularity_control()
---
c.超时机制应用
a.功能说明
通过设置合理的锁等待超时时间，避免事务无限期等待，自动回滚超时事务，防止系统阻塞。
b.代码示例
---
import pymysql
import logging
import time
import threading
from contextlib import contextmanager
logger = logging.getLogger(__name__)
class TimeoutTransactionManager:
"""带超时机制的事务管理器"""
def __init__(self, connection_config, lock_wait_timeout=10):
self.connection_config = connection_config
self.lock_wait_timeout = lock_wait_timeout
@contextmanager
def transaction_with_timeout(self, custom_timeout=None):
"""带超时的事务上下文管理器"""
timeout = custom_timeout or self.lock_wait_timeout
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
# 设置锁等待超时
cursor.execute(f"SET SESSION innodb_lock_wait_timeout = {timeout}")
logger.info(f"设置锁等待超时: {timeout} 秒")
conn.begin()
start_time = time.time()
yield conn
elapsed = time.time() - start_time
logger.info(f"事务执行完成，耗时: {elapsed:.2f} 秒")
conn.commit()
except pymysql.err.OperationalError as e:
if 'Lock wait timeout exceeded' in str(e):
logger.warning(f"事务锁等待超时: {timeout} 秒")
conn.rollback()
raise TimeoutError(f"事务锁等待超时 ({timeout} 秒)")
elif 'deadlock' in str(e).lower():
logger.warning(f"事务遇到死锁: {e}")
conn.rollback()
raise Exception(f"事务死锁: {e}")
else:
logger.error(f"事务操作错误: {e}")
conn.rollback()
raise
except Exception as e:
conn.rollback()
logger.error(f"事务执行失败: {e}")
raise
finally:
cursor.close()
conn.close()
def execute_with_retry(self, operation_func, max_retries=3, retry_delay=1):
"""带重试机制的事务执行"""
for attempt in range(max_retries):
try:
logger.info(f"执行事务尝试 {attempt + 1}/{max_retries}")
with self.transaction_with_timeout() as conn:
return operation_func(conn)
except TimeoutError as e:
logger.warning(f"事务超时 (尝试 {attempt + 1}): {e}")
if attempt < max_retries - 1:
# 指数退避延迟
delay = retry_delay * (2 ** attempt) + random.uniform(0, 1)
logger.info(f"等待 {delay:.2f} 秒后重试")
time.sleep(delay)
continue
else:
logger.error(f"事务在 {max_retries} 次尝试后仍然超时")
raise
except Exception as e:
if 'deadlock' in str(e).lower():
logger.warning(f"事务死锁 (尝试 {attempt + 1}): {e}")
if attempt < max_retries - 1:
delay = retry_delay * (2 ** attempt) + random.uniform(0, 1)
logger.info(f"死锁重试延迟: {delay:.2f} 秒")
time.sleep(delay)
continue
else:
logger.error(f"事务在 {max_retries} 次尝试后仍然遇到死锁")
raise
else:
logger.error(f"事务执行错误: {e}")
raise
def transfer_with_timeout_protection(self, from_account, to_account, amount, timeout=5):
"""带超时保护的转账操作"""
def transfer_operation(conn):
cursor = conn.cursor()
try:
# 获取转出账户信息并锁定
cursor.execute(
"SELECT id, balance FROM accounts WHERE account_number = %s FOR UPDATE",
(from_account,)
)
from_result = cursor.fetchone()
if not from_result:
raise Exception(f"转出账户 {from_account} 不存在")
from_id, from_balance = from_result
if float(from_balance) < amount:
raise Exception(f"账户 {from_account} 余额不足")
# 获取转入账户信息
cursor.execute(
"SELECT id FROM accounts WHERE account_number = %s",
(to_account,)
)
to_result = cursor.fetchone()
if not to_result:
raise Exception(f"转入账户 {to_account} 不存在")
to_id = to_result[0]
# 执行转账
cursor.execute(
"UPDATE accounts SET balance = balance - %s WHERE id = %s",
(amount, from_id)
)
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE id = %s",
(amount, to_id)
)
# 记录交易
cursor.execute(
"""INSERT INTO transactions
(from_account, to_account, amount, transaction_type, status, processed_at)
VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP)""",
(from_account, to_account, amount, 'TRANSFER', 'completed')
)
return {
'from_account': from_account,
'to_account': to_account,
'amount': amount,
'new_from_balance': float(from_balance) - amount,
'transaction_id': cursor.lastrowid
}
finally:
cursor.close()
try:
return self.execute_with_retry(
lambda conn: transfer_operation(conn),
max_retries=3,
retry_delay=0.5
)
except Exception as e:
logger.error(f"带超时保护的转账失败: {e}")
return {'status': 'failed', 'error': str(e)}
def demonstrate_timeout_mechanism():
"""演示超时机制"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4',
'autocommit': False
}
# 准备测试数据
conn = pymysql.Connect(**connection_config)
cursor = conn.cursor()
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'TIMEOUT%'")
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[
("TIMEOUT001", "超时测试账户1", 2000.00),
("TIMEOUT002", "超时测试账户2", 1000.00)
]
)
conn.commit()
cursor.close()
conn.close()
timeout_manager = TimeoutTransactionManager(connection_config, lock_wait_timeout=3)
def concurrent_transfer_test():
"""并发转账测试，可能触发超时"""
def transfer_worker(worker_id, from_account, to_account, amount, delay=0):
logger.info(f"工作线程 {worker_id} 开始转账: {from_account} -> {to_account}")
if delay > 0:
time.sleep(delay)
result = timeout_manager.transfer_with_timeout_protection(
from_account, to_account, amount, timeout=2
)
if 'status' in result and result['status'] == 'failed':
logger.warning(f"工作线程 {worker_id} 转账失败: {result['error']}")
else:
logger.info(f"工作线程 {worker_id} 转账成功")
return result
# 启动多个并发转账
threads = []
# 第一组转账：同时锁定相同账户，可能产生冲突
t1 = threading.Thread(target=transfer_worker, args=(1, "TIMEOUT001", "TIMEOUT002", 500, 0))
t2 = threading.Thread(target=transfer_worker, args=(2, "TIMEOUT002", "TIMEOUT001", 300, 0.1))
# 第二组转账：稍微错开时间
t3 = threading.Thread(target=transfer_worker, args=(3, "TIMEOUT001", "TIMEOUT002", 200, 0.5))
t4 = threading.Thread(target=transfer_worker, args=(4, "TIMEOUT002", "TIMEOUT001", 100, 0.6))
threads.extend([t1, t2, t3, t4])
# 启动所有线程
for thread in threads:
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
logger.info("并发转账测试完成")
try:
logger.info("=== 开始超时机制演示 ===")
# 测试超时保护
logger.info("测试1: 正常转账")
result1 = timeout_manager.transfer_with_timeout_protection(
"TIMEOUT001", "TIMEOUT002", 100, timeout=10
)
print(f"正常转账结果: {result1}")
# 测试并发冲突和超时
logger.info("测试2: 并发转账（可能触发超时或死锁）")
concurrent_transfer_test()
logger.info("超时机制演示完成")
except Exception as e:
logger.error(f"超时机制演示失败: {e}")
# 运行演示
demonstrate_timeout_mechanism()
---
04.死锁恢复与处理
a.死锁检测与恢复
a.功能说明
实现自动死锁检测、记录和分析机制，当检测到死锁时自动采取恢复措施，并记录详细信息用于后续分析。
b.代码示例
---
import pymysql
import logging
import threading
import time
import json
from datetime import datetime
from queue import Queue
logger = logging.getLogger(__name__)
class DeadlockRecoveryManager:
"""死锁恢复管理器"""
def __init__(self, connection_config):
self.connection_config = connection_config
self.deadlock_queue = Queue()
self.recovery_stats = {
'total_deadlocks': 0,
'successful_recoveries': 0,
'failed_recoveries': 0,
'recovery_attempts': []
}
self.monitoring_active = False
def start_monitoring(self):
"""启动死锁监控"""
if self.monitoring_active:
logger.warning("死锁监控已在运行")
return
self.monitoring_active = True
monitor_thread = threading.Thread(target=self._monitor_deadlocks, daemon=True)
monitor_thread.start()
logger.info("死锁监控已启动")
def stop_monitoring(self):
"""停止死锁监控"""
self.monitoring_active = False
logger.info("死锁监控已停止")
def _monitor_deadlocks(self):
"""监控死锁的后台线程"""
while self.monitoring_active:
try:
# 检查最近的死锁
deadlocks = self._check_recent_deadlocks()
for deadlock in deadlocks:
self.deadlock_queue.put(deadlock)
self.recovery_stats['total_deadlocks'] += 1
logger.info(f"检测到死锁: {deadlock['id']}")
time.sleep(5) # 每5秒检查一次
except Exception as e:
logger.error(f"死锁监控出错: {e}")
time.sleep(10) # 出错后等待更长时间
def _check_recent_deadlocks(self):
"""检查最近的死锁"""
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
try:
# 查询InnoDB状态
cursor.execute("SHOW ENGINE INNODB STATUS")
result = cursor.fetchone()
# 解析死锁信息
status_text = result[0] if result else ""
deadlocks = self._parse_innodb_status(status_text)
return deadlocks
except Exception as e:
logger.error(f"检查死锁失败: {e}")
return []
finally:
cursor.close()
conn.close()
def _parse_innodb_status(self, status_text):
"""解析InnoDB状态中的死锁信息"""
deadlocks = []
lines = status_text.split('\n')
current_deadlock = None
in_deadlock_section = False
for line in lines:
if 'LATEST DETECTED DEADLOCK' in line:
in_deadlock_section = True
current_deadlock = {
'id': len(self.recovery_stats['recovery_attempts']) + 1,
'timestamp': datetime.now().isoformat(),
'transactions': [],
'locks': []
}
continue
if in_deadlock_section:
if '------------------------' in line:
if current_deadlock:
deadlocks.append(current_deadlock)
in_deadlock_section = False
current_deadlock = None
continue
if current_deadlock:
if 'TRANSACTION' in line:
transaction_info = self._parse_transaction_line(line)
if transaction_info:
current_deadlock['transactions'].append(transaction_info)
elif 'RECORD LOCKS' in line or 'TABLE LOCK' in line:
lock_info = self._parse_lock_line(line)
if lock_info:
current_deadlock['locks'].append(lock_info)
if current_deadlock and in_deadlock_section:
deadlocks.append(current_deadlock)
return deadlocks
def _parse_transaction_line(self, line):
"""解析事务信息行"""
try:
if 'TRANSACTION' in line:
# 提取事务ID和状态
parts = line.split()
for i, part in enumerate(parts):
if part == 'TRANSACTION' and i + 1 < len(parts):
return {
'id': parts[i + 1],
'line': line.strip(),
'type': 'transaction'
}
except Exception:
pass
return None
def _parse_lock_line(self, line):
"""解析锁信息行"""
try:
if any(keyword in line.upper() for keyword in ['LOCK', 'RECORD']):
return {
'line': line.strip(),
'type': 'lock'
}
except Exception:
pass
return None
def handle_deadlock_recovery(self, deadlock):
"""处理死锁恢复"""
recovery_start = time.time()
attempt_id = len(self.recovery_stats['recovery_attempts'])
recovery_attempt = {
'attempt_id': attempt_id,
'deadlock_id': deadlock['id'],
'start_time': recovery_start,
'strategies_tried': []
}
logger.info(f"开始死锁恢复: 尝试 {attempt_id}, 死锁 {deadlock['id']}")
try:
# 策略1: 等待系统自动解决
if self._try_automatic_resolution(deadlock, recovery_attempt):
success = True
else:
# 策略2: 手动干预（如果有必要）
success = self._try_manual_intervention(deadlock, recovery_attempt)
recovery_attempt['end_time'] = time.time()
recovery_attempt['duration'] = recovery_attempt['end_time'] - recovery_attempt['start_time']
recovery_attempt['success'] = success
self.recovery_stats['recovery_attempts'].append(recovery_attempt)
if success:
self.recovery_stats['successful_recoveries'] += 1
logger.info(f"死锁恢复成功: 尝试 {attempt_id}, 耗时 {recovery_attempt['duration']:.2f} 秒")
else:
self.recovery_stats['failed_recoveries'] += 1
logger.error(f"死锁恢复失败: 尝试 {attempt_id}")
return success
except Exception as e:
recovery_attempt['end_time'] = time.time()
recovery_attempt['duration'] = recovery_attempt['end_time'] - recovery_attempt['start_time']
recovery_attempt['success'] = False
recovery_attempt['error'] = str(e)
self.recovery_stats['recovery_attempts'].append(recovery_attempt)
self.recovery_stats['failed_recoveries'] += 1
logger.error(f"死锁恢复异常: 尝试 {attempt_id}, 错误: {e}")
return False
def _try_automatic_resolution(self, deadlock, recovery_attempt):
"""尝试自动解决方案"""
logger.info("尝试自动死锁解决")
recovery_attempt['strategies_tried'].append('automatic_resolution')
# 等待一小段时间让MySQL自动处理死锁
time.sleep(2)
# 检查死锁是否已解决
remaining_deadlocks = self._check_recent_deadlocks()
current_deadlock_ids = [d['id'] for d in remaining_deadlocks]
if deadlock['id'] not in current_deadlock_ids:
logger.info("死锁已自动解决")
return True
else:
logger.warning("死锁未自动解决")
return False
def _try_manual_intervention(self, deadlock, recovery_attempt):
"""尝试手动干预"""
logger.info("尝试手动干预解决死锁")
recovery_attempt['strategies_tried'].append('manual_intervention')
try:
conn = pymysql.Connect(**self.connection_config)
cursor = conn.cursor()
# 查询当前锁等待情况
cursor.execute("""
SELECT
r.trx_id AS waiting_trx_id,
r.trx_mysql_thread_id AS waiting_thread,
b.trx_id AS blocking_trx_id,
b.trx_mysql_thread_id AS blocking_thread,
l.lock_table
FROM information_schema.innodb_lock_waits w
INNER JOIN information_schema.innodb_trx r ON r.trx_id = w.requesting_trx_id
INNER JOIN information_schema.innodb_trx b ON b.trx_id = w.blocking_trx_id
INNER JOIN information_schema.innodb_locks l ON l.lock_trx_id = w.blocking_trx_id
LIMIT 10
""")
lock_waits = cursor.fetchall()
logger.info(f"当前锁等待情况: {len(lock_waits)} 个")
# 策略: 终止等待时间最长的事务
if lock_waits:
# 查询事务等待时间
cursor.execute("""
SELECT
id,
time,
info,
state
FROM information_schema.processlist
WHERE id IN (
SELECT trx_mysql_thread_id
FROM information_schema.innodb_lock_waits
UNION
SELECT blocking_trx_mysql_thread_id
FROM information_schema.innodb_lock_waits
)
ORDER BY time DESC
LIMIT 1
""")
longest_waiting = cursor.fetchone()
if longest_waiting and longest_waiting[1] > 30: # 等待超过30秒
thread_id = longest_waiting[0]
logger.warning(f"终止等待时间过长的事务: 线程 {thread_id}")
try:
cursor.execute(f"KILL {thread_id}")
logger.info(f"已终止线程 {thread_id}")
time.sleep(2) # 等待清理
return True
except Exception as kill_error:
logger.error(f"终止线程失败: {kill_error}")
cursor.close()
conn.close()
except Exception as e:
logger.error(f"手动干预失败: {e}")
return False
def generate_recovery_report(self):
"""生成死锁恢复报告"""
report = {
'report_time': datetime.now().isoformat(),
'statistics': {
'total_deadlocks': self.recovery_stats['total_deadlocks'],
'successful_recoveries': self.recovery_stats['successful_recoveries'],
'failed_recoveries': self.recovery_stats['failed_recoveries'],
'success_rate': (
self.recovery_stats['successful_recoveries'] /
max(1, self.recovery_stats['total_deadlocks'])
) * 100
},
'recovery_attempts': self.recovery_stats['recovery_attempts'],
'recommendations': self._generate_recovery_recommendations()
}
return report
def _generate_recovery_recommendations(self):
"""生成恢复建议"""
recommendations = []
stats = self.recovery_stats
if stats['failed_recoveries'] > stats['successful_recoveries']:
recommendations.append({
'priority': 'high',
'category': 'system_configuration',
'issue': '死锁恢复失败率过高',
'recommendation': '检查系统配置和应用程序代码',
'action': '优化事务逻辑，减少锁竞争'
})
if stats['total_deadlocks'] > 10:
recommendations.append({
'priority': 'medium',
'category': 'performance_optimization',
'issue': f'死锁频率过高 ({stats["total_deadlocks"]} 次)',
'recommendation': '优化数据库设计和查询',
'action': '考虑使用乐观锁、读写分离或缓存'
})
avg_recovery_time = 0
if stats['recovery_attempts']:
successful_attempts = [a for a in stats['recovery_attempts'] if a.get('success')]
if successful_attempts:
avg_recovery_time = sum(a.get('duration', 0) for a in successful_attempts) / len(successful_attempts)
if avg_recovery_time > 30:
recommendations.append({
'priority': 'medium',
'category': 'response_time',
'issue': f'平均恢复时间过长 ({avg_recovery_time:.1f} 秒)',
'recommendation': '优化恢复策略',
'action': '实现更快的死锁检测和解决机制'
})
return recommendations
def demonstrate_deadlock_recovery():
"""演示死锁恢复机制"""
connection_config = {
'host': 'localhost',
'user': 'root',
'password': 'password',
'database': 'bank_db',
'charset': 'utf8mb4'
}
recovery_manager = DeadlockRecoveryManager(connection_config)
try:
logger.info("=== 开始死锁恢复演示 ===")
# 启动死锁监控
recovery_manager.start_monitoring()
# 准备测试数据
conn = pymysql.Connect(**connection_config)
cursor = conn.cursor()
cursor.execute("DELETE FROM accounts WHERE account_number LIKE 'RECOVERY%'")
cursor.executemany(
"""INSERT INTO accounts (account_number, account_name, balance)
VALUES (%s, %s, %s)""",
[
("RECOVERY001", "恢复测试账户1", 2000.00),
("RECOVERY002", "恢复测试账户2", 1500.00),
("RECOVERY003", "恢复测试账户3", 1000.00)
]
)
conn.commit()
cursor.close()
conn.close()
# 模拟一些可能产生死锁的操作
def simulate_potential_deadlock():
"""模拟可能产生死锁的操作"""
def worker_with_recovery(worker_id):
conn = pymysql.Connect(**connection_config)
cursor = conn.cursor()
try:
conn.begin()
logger.info(f"恢复测试线程 {worker_id} 开始")
# 执行可能冲突的操作
operations = [
("RECOVERY001", "RECOVERY002", 100),
("RECOVERY002", "RECOVERY003", 50),
("RECOVERY003", "RECOVERY001", 75)
]
for from_acct, to_acct, amount in operations:
try:
# 锁定转出账户
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = %s FOR UPDATE",
(from_acct,)
)
# 模拟处理延迟
time.sleep(0.5)
# 锁定转入账户
cursor.execute(
"SELECT balance FROM accounts WHERE account_number = %s FOR UPDATE",
(to_acct,)
)
# 执行转账
cursor.execute(
"UPDATE accounts SET balance = balance - %s WHERE account_number = %s",
(amount, from_acct)
)
cursor.execute(
"UPDATE accounts SET balance = balance + %s WHERE account_number = %s",
(amount, to_acct)
)
logger.info(f"线程 {worker_id} 转账成功: {from_acct} -> {to_acct}")
except Exception as e:
logger.warning(f"线程 {worker_id} 操作失败: {e}")
break
conn.commit()
logger.info(f"恢复测试线程 {worker_id} 完成")
except Exception as e:
conn.rollback()
logger.error(f"恢复测试线程 {worker_id} 失败: {e}")
finally:
cursor.close()
conn.close()
# 启动多个工作线程
threads = []
for i in range(5):
thread = threading.Thread(target=worker_with_recovery, args=(i+1,))
threads.append(thread)
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 运行模拟
simulate_potential_deadlock()
# 等待监控处理死锁
logger.info("等待死锁监控处理...")
time.sleep(10)
# 停止监控
recovery_manager.stop_monitoring()
# 生成恢复报告
report = recovery_manager.generate_recovery_report()
print("=== 死锁恢复报告 ===")
print(json.dumps(report, indent=2, ensure_ascii=False))
if report['recommendations']:
print("\n=== 恢复建议 ===")
for rec in report['recommendations']:
print(f"[{rec['priority'].upper()}] {rec['category']}")
print(f"问题: {rec['issue']}")
print(f"建议: {rec['recommendation']}")
print(f"行动: {rec['action']}\n")
except Exception as e:
logger.error(f"死锁恢复演示失败: {e}")
finally:
recovery_manager.stop_monitoring()
# 运行演示
demonstrate_deadlock_recovery()
---
3.5 事务重试机制
01.重试机制概述
a.基本概念
事务重试机制是指在事务执行过程中遇到可恢复错误时，系统自动重新执行事务的策略。这种机制主要用于处理数据库操作中常见的瞬时错误，如死锁、连接超时、网络抖动等问题。
b.应用场景
高并发环境下的数据库操作经常遇到资源竞争和临时不可用情况，重试机制能够显著提高系统的可用性和用户体验。特别是在分布式系统和微服务架构中，网络不稳定和资源竞争是常态，重试机制成为保证事务完整性的重要手段。
c.设计原则
重试机制设计需要遵循幂等性、有限次数、递增延迟和错误分类等核心原则，确保重试过程不会导致数据不一致或无限循环。
02.重试策略类型
a.固定间隔重试
采用固定的时间间隔进行重试，实现简单但可能在系统负载较高时加重问题。适用于轻量级错误和快速恢复场景，间隔时间通常设置为几秒到几十秒之间。
b.指数退避重试
每次重试间隔时间按指数增长，能够有效避免雪崩效应。初始间隔较短，后续间隔逐渐增加，给系统提供足够的恢复时间，是最常用的重试策略之一。
c.随机抖动重试
在指数退避基础上增加随机因子，避免多个客户端同时重试造成的同步问题。通过引入随机性，将重试时间分散到不同时间点，减少系统压力峰值。
03.错误分类处理
a.可重试错误识别
包括死锁、连接超时、临时锁定等瞬时错误，这些错误在重试后通常能够自行解决。数据库系统会自动释放资源，网络连接会恢复，因此重试是合理的处理方式。
b.错误代码判断
通过错误代码和异常信息区分可重试和不可重试错误。MySQL死锁错误代码1213，连接超时错误代码2003，语法错误代码1064等，需要建立错误代码与重试策略的映射关系。
c.重试决策逻辑
根据错误类型、系统状态、业务重要性等因素动态判断是否重试。对于高优先级业务可以适当增加重试次数，对于非关键业务可以快速失败。
04.死锁重试处理
a.死锁检测机制
数据库管理系统通过死锁检测算法识别循环等待的资源竞争关系，选择牺牲成本最小的事务进行回滚。MySQL使用等待图检测死锁，Oracle使用超时机制检测死锁。
b.死锁错误识别
MySQL死锁异常信息通常包含"Deadlock found"和错误代码1213，应用程序需要捕获这些特定异常并进行重试处理。同时记录死锁发生的事务ID和涉及的数据资源。
c.基础重试装饰器
---
# 死锁重试装饰器示例
import time
import random
import functools
import pymysql
from pymysql import MySQLError
def retry_on_deadlock(max_retries=3, base_delay=0.1):
def decorator(func):
@functools.wraps(func)
def wrapper(*args, **kwargs):
last_exception = None
for attempt in range(max_retries + 1):
try:
return func(*args, **kwargs)
except MySQLError as e:
# 检查是否为死锁错误
if e.args[0] == 1213 and attempt < max_retries:
delay = base_delay * (2 ** attempt) + random.uniform(0, 0.1)
print(f"死锁重试第{attempt + 1}次，等待{delay:.2f}秒")
time.sleep(delay)
last_exception = e
continue
else:
raise e
raise last_exception
return wrapper
return decorator
---
05.连接失败重试
a.连接池管理
使用连接池可以减少连接建立开销，提高重试效率。连接池需要配置合理的最大连接数、超时时间和空闲连接回收策略，确保在重试过程中能够获取到可用连接。
b.连接错误处理
MySQL连接错误包括2003(无法连接)、2006(服务器消失)、2013(查询期间失去连接)等。针对不同的连接错误采用不同的重试策略，如网络问题使用指数退避，认证问题直接失败。
c.连接重试实现
---
# 连接重试管理器
class ConnectionRetryManager:
def __init__(self, max_retries=5, base_delay=1.0):
self.max_retries = max_retries
self.base_delay = base_delay
self.retryable_errors = [2003, 2006, 2013] # 可重试的MySQL连接错误
def is_retryable_error(self, error_code):
"""判断错误是否可以重试"""
return error_code in self.retryable_errors
def get_delay(self, attempt):
"""计算重试延迟时间"""
return self.base_delay * (2 ** attempt) + random.uniform(0, 0.5)
def connect_with_retry(self, connection_params):
"""带重试的连接方法"""
last_exception = None
for attempt in range(self.max_retries + 1):
try:
connection = pymysql.connect(**connection_params)
# 验证连接可用性
with connection.cursor() as cursor:
cursor.execute("SELECT 1")
print(f"连接成功，尝试次数: {attempt + 1}")
return connection
except MySQLError as e:
error_code = e.args[0] if e.args else 0
if not self.is_retryable_error(error_code) or attempt == self.max_retries:
raise e
delay = self.get_delay(attempt)
print(f"连接重试第{attempt + 1}次，等待{delay:.2f}秒，错误: {error_code}")
time.sleep(delay)
last_exception = e
raise last_exception
---
d.连接验证机制
在重试前验证连接的有效性，避免使用已断开的连接。可以通过心跳检测、简单查询或ping命令验证连接状态，确保重试使用的是健康连接。
06.重试控制参数
a.最大重试次数
设置合理的重试次数上限，通常3-5次较为合适。过多的重试会消耗系统资源，过少的重试可能无法解决问题。需要根据错误类型和业务重要性调整重试次数。
b.重试间隔配置
初始间隔、最大间隔、退避因子等参数需要根据系统特性和负载情况调整。初始间隔通常1-5秒，最大间隔30-60秒，退避因子2-3较为常见。
c.通用重试配置类
---
# 重试配置管理
from dataclasses import dataclass
from typing import List, Optional
@dataclass
class RetryConfig:
max_retries: int = 3
base_delay: float = 1.0
max_delay: float = 60.0
backoff_factor: float = 2.0
jitter: bool = True
retryable_errors: List[int] = None
timeout: Optional[float] = None
def __post_init__(self):
if self.retryable_errors is None:
self.retryable_errors = [1213, 2003, 2006, 2013] # 默认可重试错误
def calculate_delay(self, attempt: int) -> float:
"""计算第attempt次重试的延迟时间"""
delay = self.base_delay * (self.backoff_factor ** attempt)
delay = min(delay, self.max_delay)
if self.jitter:
# 添加随机抖动，避免同步重试
jitter_range = delay * 0.1
delay += random.uniform(-jitter_range, jitter_range)
return max(0, delay)
def should_retry(self, error_code: int, attempt: int) -> bool:
"""判断是否应该重试"""
return (error_code in self.retryable_errors and
attempt < self.max_retries)
---
d.重试超时控制
设置总体重试时间上限，避免重试过程无限期持续。当累计重试时间超过阈值时，应该终止重试并记录错误，防止系统长时间处于不确定状态。
07.重试监控与日志
a.重试指标收集
记录重试次数、重试成功率、重试耗时等关键指标，用于系统监控和性能分析。通过指标分析可以识别系统瓶颈和优化重试策略。
b.详细日志记录
记录每次重试的错误信息、时间戳、重试间隔等详细信息，便于问题排查和系统优化。日志级别应该可配置，避免在高负载环境下产生过多日志。
c.告警机制设置
当重试次数或重试失败率超过阈值时触发告警，提醒运维人员关注系统状态。告警规则应该根据业务重要性分级，避免告警风暴。
08.最佳实践建议
a.幂等性设计
确保重试操作不会产生副作用，这是重试机制的前提条件。可以通过唯一键、版本号、状态机等方式保证操作的幂等性。
b.事务重试上下文管理器
---
# 完整的事务重试管理器
class TransactionRetryManager:
def __init__(self, config: RetryConfig = None):
self.config = config or RetryConfig()
self.retry_stats = {
'total_retries': 0,
'successful_retries': 0,
'failed_retries': 0
}
def execute_with_retry(self, func, *args, **kwargs):
"""执行带重试的事务操作"""
start_time = time.time()
last_exception = None
for attempt in range(self.config.max_retries + 1):
try:
if attempt > 0:
delay = self.config.calculate_delay(attempt - 1)
print(f"重试第{attempt}次，等待{delay:.2f}秒")
time.sleep(delay)
result = func(*args, **kwargs)
if attempt > 0:
self.retry_stats['successful_retries'] += 1
print(f"重试成功，总重试次数: {attempt}")
return result
except MySQLError as e:
error_code = e.args[0] if e.args else 0
if not self.config.should_retry(error_code, attempt):
self.retry_stats['failed_retries'] += 1
raise e
if attempt < self.config.max_retries:
self.retry_stats['total_retries'] += 1
print(f"可重试错误 {error_code}，继续重试")
last_exception = e
else:
self.retry_stats['failed_retries'] += 1
raise e
except Exception as e:
# 非数据库错误，直接抛出
self.retry_stats['failed_retries'] += 1
raise e
# 检查超时
if self.config.timeout and (time.time() - start_time) > self.config.timeout:
print(f"重试超时，已用时: {time.time() - start_time:.2f}秒")
raise TimeoutError("重试操作超时")
raise last_exception
def get_stats(self):
"""获取重试统计信息"""
return self.retry_stats.copy()
def reset_stats(self):
"""重置统计信息"""
self.retry_stats = {
'total_retries': 0,
'successful_retries': 0,
'failed_retries': 0
}
---
c.重试与补偿结合
对于复杂的分布式事务，重试机制应该与补偿机制结合使用。当重试最终失败时，通过补偿操作回滚已完成的步骤，保证数据一致性。补偿操作需要保证幂等性，可以多次执行而不产生副作用。
09.实际应用示例
a.订单处理事务重试
---
# 订单处理中的事务重试应用
import logging
from contextlib import contextmanager
class OrderService:
def __init__(self, db_connection):
self.db = db_connection
self.retry_manager = TransactionRetryManager(
RetryConfig(
max_retries=3,
base_delay=0.5,
max_delay=10.0,
backoff_factor=2.0,
timeout=30.0
)
)
@retry_on_deadlock(max_retries=3, base_delay=0.1)
def create_order_with_retry(self, user_id, product_id, quantity):
"""创建订单，自动处理死锁重试"""
return self._create_order(user_id, product_id, quantity)
def _create_order(self, user_id, product_id, quantity):
"""实际创建订单的业务逻辑"""
try:
with self.db.cursor() as cursor:
# 开始事务
self.db.begin()
# 检查库存并锁定
cursor.execute(
"SELECT stock FROM products WHERE id = %s FOR UPDATE",
(product_id,)
)
result = cursor.fetchone()
if not result or result[0] < quantity:
self.db.rollback()
raise ValueError("库存不足")
# 扣减库存
cursor.execute(
"UPDATE products SET stock = stock - %s WHERE id = %s",
(quantity, product_id)
)
# 创建订单
order_amount = self._calculate_amount(product_id, quantity)
cursor.execute(
"INSERT INTO orders (user_id, product_id, quantity, amount) "
"VALUES (%s, %s, %s, %s)",
(user_id, product_id, quantity, order_amount)
)
order_id = cursor.lastrowid
# 提交事务
self.db.commit()
logging.info(f"订单创建成功，订单ID: {order_id}")
return order_id
except Exception as e:
self.db.rollback()
logging.error(f"订单创建失败: {e}")
raise
def _calculate_amount(self, product_id, quantity):
"""计算订单金额"""
with self.db.cursor() as cursor:
cursor.execute("SELECT price FROM products WHERE id = %s", (product_id,))
result = cursor.fetchone()
if not result:
raise ValueError("商品不存在")
return result[0] * quantity
def batch_create_orders(self, orders_data):
"""批量创建订单，使用重试管理器"""
created_orders = []
failed_orders = []
for i, order_data in enumerate(orders_data):
try:
order_id = self.retry_manager.execute_with_retry(
self.create_order_with_retry,
order_data['user_id'],
order_data['product_id'],
order_data['quantity']
)
created_orders.append(order_id)
except Exception as e:
failed_orders.append({
'index': i,
'data': order_data,
'error': str(e)
})
logging.error(f"订单创建失败: {order_data}, 错误: {e}")
# 输出重试统计
stats = self.retry_manager.get_stats()
logging.info(f"批量订单完成，重试统计: {stats}")
return created_orders, failed_orders
---
b.重试监控集成
---
# 重试监控和告警集成
class RetryMonitor:
def __init__(self, alert_threshold=10):
self.alert_threshold = alert_threshold
self.failure_count = 0
self.retry_count = 0
def record_retry(self, error_code, attempt_count):
"""记录重试事件"""
self.retry_count += 1
if error_code == 1213: # 死锁
logging.warning(f"死锁重试记录，重试次数: {attempt_count}")
if self.retry_count >= self.alert_threshold:
self.trigger_alert()
def record_failure(self):
"""记录重试失败"""
self.failure_count += 1
if self.failure_count >= self.alert_threshold:
self.trigger_alert()
def trigger_alert(self):
"""触发告警"""
alert_message = (
f"重试告警: 失败次数={self.failure_count}, "
f"重试次数={self.retry_count}, 阈值={self.alert_threshold}"
)
logging.error(alert_message)
# 这里可以集成实际的告警系统，如邮件、短信等
---
c.使用建议总结
在实际项目中使用重试机制时，建议根据业务场景调整重试参数，对于关键业务可以增加重试次数和超时时间。同时要建立完善的监控体系，及时发现和处理重试异常情况。
4. PostgreSQL事务
4.1 psycopg2模块
01.psycopg2简介
a.基本概念
psycopg2是Python中最流行的PostgreSQL数据库适配器，提供了完整的数据库连接、查询和事务管理功能。它支持服务端游标、异步操作、COPY命令等高级特性，并且完全兼容DB-API 2.0规范。
b.核心特性
支持连接池管理、异步操作、二进制数据传输、服务器端游标、COPY命令等高级功能。psycopg2还支持预编译语句、类型适配器和连接状态管理，为PostgreSQL事务处理提供了强大的基础支持。
c.安装配置
通过pip install psycopg2-binary快速安装，支持PostgreSQL 9.1及以上版本。配置时需要设置正确的连接参数，包括主机名、端口、数据库名、用户名和密码等基本信息。
02.连接管理
a.基本连接方式
使用psycopg2.connect()建立与PostgreSQL数据库的连接，支持多种连接字符串格式和参数配置。连接对象提供了事务控制方法，包括commit()、rollback()和autocommit属性设置。
b.连接池管理
通过连接池可以有效管理数据库连接，避免频繁建立和断开连接的开销。psycopg2支持第三方连接池库如psycopg2-pool，也可以自定义连接池管理逻辑。
c.连接状态监控
使用connection对象的status、closed等属性监控连接状态，及时发现和处理连接异常问题。还可以设置连接超时、查询超时等参数提高连接可靠性。
03.基础事务操作
a.事务开始
PostgreSQL默认开启事务模式，所有操作都在事务中执行。可以设置connection.autocommit = True来启用自动提交模式，但通常建议使用显式事务控制。
b.事务提交与回滚
使用connection.commit()提交事务，将所有修改永久保存到数据库。使用connection.rollback()回滚事务，撤销所有未提交的修改操作。
c.事务隔离级别
PostgreSQL支持多种隔离级别：READ UNCOMMITTED、READ COMMITTED、REPEATABLE READ和SERIALIZABLE。可以通过SET TRANSACTION命令或connection对象的isolation_level属性设置隔离级别。
04.游标管理
a.服务端游标
服务端游标允许在服务器端处理大型结果集，减少内存占用。使用name参数创建命名游标，支持fetchone()、fetchmany()、fetchall()等方法获取数据。
b.客户端游标
默认游标类型，一次性将所有结果加载到客户端内存。适用于小型结果集，访问速度快但占用内存较多。
c.游标关闭与清理
使用完游标后必须调用close()方法关闭，释放服务器资源。推荐使用with语句自动管理游标生命周期，确保资源正确释放。
05.错误处理
a.异常类型
psycopg2定义了多种异常类型，包括OperationalError、IntegrityError、ProgrammingError等。不同异常类型对应不同的错误情况，便于进行精确的错误处理。
b.错误代码映射
PostgreSQL错误代码与SQLSTATE标准兼容，可以通过异常对象的pgcode属性获取错误代码。常见的错误代码包括23505(唯一约束违反)、23503(外键约束违反)等。
c.异常恢复策略
根据异常类型采取不同的恢复策略，对于连接错误需要重新连接，对于约束错误需要检查数据完整性，对于死锁错误需要实现重试机制。
06.基础操作示例
a.连接与事务控制
---
# PostgreSQL基础操作示例
import psycopg2
import psycopg2.extras
from psycopg2.errors import OperationalError, IntegrityError
def basic_database_operations():
"""基础数据库操作示例"""
try:
# 建立连接
conn = psycopg2.connect(
host="localhost",
database="testdb",
user="postgres",
password="password",
port=5432
)
with conn.cursor() as cursor:
# 开始事务
print("开始事务...")
cursor.execute("BEGIN")
# 创建表
cursor.execute("""
CREATE TABLE IF NOT EXISTS users (
id SERIAL PRIMARY KEY,
name VARCHAR(50) NOT NULL,
email VARCHAR(100) UNIQUE NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP
)
""")
print("创建表成功")
# 插入数据
cursor.execute("""
INSERT INTO users (name, email) VALUES
(%s, %s), (%s, %s)
RETURNING id
""", ("张三", "[email protected]", "李四", "[email protected]"))
# 获取插入的ID
result = cursor.fetchall()
print(f"插入用户ID: {[row[0] for row in result]}")
# 查询数据
cursor.execute("SELECT * FROM users")
users = cursor.fetchall()
for user in cursor.fetchall():
print(f"用户: {user}")
# 提交事务
cursor.execute("COMMIT")
print("事务提交成功")
except IntegrityError as e:
print(f"数据完整性错误: {e}")
conn.rollback()
except OperationalError as e:
print(f"操作错误: {e}")
except Exception as e:
print(f"未知错误: {e}")
finally:
if 'conn' in locals():
conn.close()
---
b.事务回滚示例
---
# 事务回滚示例
def transaction_rollback_example():
"""事务回滚示例"""
conn = psycopg2.connect(
host="localhost",
database="testdb",
user="postgres",
password="password"
)
try:
with conn.cursor() as cursor:
cursor.execute("BEGIN")
# 尝试插入重复的数据，会触发约束违反
cursor.execute(
"INSERT INTO users (name, email) VALUES (%s, %s)",
("王五", "[email protected]") # 假设[email protected]已存在
)
cursor.execute("COMMIT")
print("数据插入成功")
except IntegrityError as e:
print(f"违反唯一约束: {e}")
print("执行事务回滚...")
conn.rollback()
print("事务已回滚")
# 查看回滚后的数据
with conn.cursor() as cursor:
cursor.execute("SELECT COUNT(*) FROM users WHERE email = '[email protected]'")
count = cursor.fetchone()[0]
print(f"[email protected]的记录数: {count}")
finally:
conn.close()
---
4.2 事务块
01.事务块基础
a.BEGIN命令
使用BEGIN或START TRANSACTION命令显式开始事务块，这将关闭自动提交模式，进入事务状态。事务块中的所有操作要么全部成功，要么全部失败，保证数据一致性。
b.COMMIT提交
使用COMMIT命令提交事务，将事务块中的所有操作永久保存到数据库。提交后事务结束，自动返回到自动提交模式，所有修改对其他事务可见。
c.ROLLBACK回滚
使用ROLLBACK命令回滚事务，撤销事务块中的所有操作。回滚后事务结束，数据库恢复到事务开始前的状态，所有修改都不会被保存。
02.事务控制语法
a.基础事务块
标准的事务控制流程包括BEGIN、执行SQL操作、COMMIT或ROLLBACK。可以使用SAVEPOINT创建保存点，实现部分回滚功能。
b.保存点管理
SAVEPOINT允许在事务中创建标记点，可以回滚到特定的保存点而不是整个事务。这对于复杂的业务逻辑处理非常有用，可以精细控制回滚范围。
c.释放保存点
使用RELEASE SAVEPOINT释放不再需要的保存点，释放相关资源。ROLLBACK TO SAVEPOINT后，保存点仍然存在，可以再次使用。
03.事务状态管理
a.事务状态查询
通过查询系统视图pg_stat_activity、pg_locks等可以监控当前事务状态。还可以使用txid_current()获取当前事务ID，用于事务追踪和调试。
b.锁等待监控
PostgreSQL提供丰富的锁信息查询功能，可以监控锁等待情况，识别潜在的死锁问题。pg_locks视图显示当前所有锁信息，pg_blocking_processes显示阻塞关系。
c.超时处理
设置statement_timeout、lock_timeout等参数控制事务执行时间，避免长时间运行的查询或锁等待影响系统性能。超时后会自动回滚当前事务。
04.事务嵌套
a.子事务概念
PostgreSQL通过保存点实现类似嵌套事务的功能。虽然不支持真正的嵌套事务，但可以通过保存点实现部分回滚，达到类似的效果。
b.保存点回滚
使用ROLLBACK TO SAVEPOINT可以回滚到特定保存点，释放该保存点之后的所有修改。保存点之前的状态和操作保持不变。
c.异常处理
在复杂的事务操作中，合理使用异常处理和保存点，确保在发生错误时能够正确回滚到合适的状态，而不是整个事务回滚。
05.上下文管理器
a.with语句支持
虽然psycopg2本身不提供事务上下文管理器，但可以自定义实现事务上下文管理器，确保事务的正确提交或回滚。
b.自动资源管理
上下文管理器可以自动处理连接、游标等资源的创建和释放，避免资源泄漏问题。
c.异常安全保证
使用上下文管理器可以确保即使在发生异常的情况下，事务也能正确处理，要么提交要么回滚，避免数据不一致。
06.实现示例
a.基础事务块操作
---
# PostgreSQL事务块基础操作示例
import psycopg2
from psycopg2.errors import IntegrityError, OperationalError
from contextlib import contextmanager
@contextmanager
def transaction_context(conn):
"""事务上下文管理器"""
try:
with conn.cursor() as cursor:
cursor.execute("BEGIN")
yield conn
with conn.cursor() as cursor:
cursor.execute("COMMIT")
print("事务提交成功")
except Exception as e:
print(f"事务异常: {e}")
with conn.cursor() as cursor:
cursor.execute("ROLLBACK")
print("事务已回滚")
raise
def basic_transaction_example():
"""基础事务块示例"""
conn = psycopg2.connect(
host="localhost",
database="testdb",
user="postgres",
password="password"
)
try:
with transaction_context(conn):
with conn.cursor() as cursor:
# 创建表
cursor.execute("""
CREATE TABLE IF NOT EXISTS products (
id SERIAL PRIMARY KEY,
name VARCHAR(100) NOT NULL,
price DECIMAL(10,2) NOT NULL,
stock INTEGER DEFAULT 0
)
""")
# 插入数据
cursor.execute("""
INSERT INTO products (name, price, stock) VALUES
('笔记本电脑', 5999.99, 50),
('无线鼠标', 199.99, 200),
('机械键盘', 799.99, 100)
""")
# 查询插入的数据
cursor.execute("SELECT COUNT(*) FROM products")
count = cursor.fetchone()[0]
print(f"插入了 {count} 个产品")
except Exception as e:
print(f"操作失败: {e}")
finally:
conn.close()
---
b.保存点管理示例
---
# 保存点管理示例
def savepoint_transaction_example():
"""保存点事务示例"""
conn = psycopg2.connect(
host="localhost",
database="testdb",
user="postgres",
password="password"
)
try:
with conn.cursor() as cursor:
cursor.execute("BEGIN")
print("开始事务")
# 插入第一个产品
cursor.execute("""
INSERT INTO products (name, price, stock)
VALUES ('智能手机', 2999.99, 30)
""")
print("插入智能手机")
# 创建第一个保存点
cursor.execute("SAVEPOINT sp1")
print("创建保存点 sp1")
# 尝试插入重复的产品（可能失败）
try:
cursor.execute("""
INSERT INTO products (name, price, stock)
VALUES ('智能手机', 2999.99, 30)
""")
print("插入重复产品成功（意外）")
except IntegrityError:
print("插入重复产品失败，回滚到保存点 sp1")
cursor.execute("ROLLBACK TO SAVEPOINT sp1")
# 插入不同的产品
cursor.execute("""
INSERT INTO products (name, price, stock)
VALUES ('智能手表', 1999.99, 80)
""")
print("插入智能手表")
# 创建第二个保存点
cursor.execute("SAVEPOINT sp2")
print("创建保存点 sp2")
# 模拟另一个失败
try:
# 插入价格异常的产品
cursor.execute("""
INSERT INTO products (name, price, stock)
VALUES ('无效产品', -100, 0)
""")
except Exception as e:
print(f"插入无效产品失败: {e}")
cursor.execute("ROLLBACK TO SAVEPOINT sp2")
print("回滚到保存点 sp2")
# 释放不再需要的保存点
cursor.execute("RELEASE SAVEPOINT sp1")
print("释放保存点 sp1")
cursor.execute("COMMIT")
print("事务提交成功")
except Exception as e:
print(f"事务失败: {e}")
with conn.cursor() as cursor:
cursor.execute("ROLLBACK")
finally:
conn.close()
---
c.上下文管理器实现
---
# 高级事务上下文管理器
from typing import Optional, Any, Callable
class TransactionManager:
"""事务管理器"""
def __init__(self, connection, isolation_level=None):
self.connection = connection
self.isolation_level = isolation_level
self.savepoints = []
def __enter__(self):
self.begin()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
if exc_type is None:
self.commit()
else:
self.rollback()
return False # 不抑制异常
def begin(self):
"""开始事务"""
with self.connection.cursor() as cursor:
if self.isolation_level:
cursor.execute(f"SET TRANSACTION ISOLATION LEVEL {self.isolation_level}")
cursor.execute("BEGIN")
def commit(self):
"""提交事务"""
with self.connection.cursor() as cursor:
cursor.execute("COMMIT")
def rollback(self):
"""回滚事务"""
with self.connection.cursor() as cursor:
cursor.execute("ROLLBACK")
def savepoint(self, name: Optional[str] = None) -> str:
"""创建保存点"""
if name is None:
name = f"sp_{len(self.savepoints)}"
with self.connection.cursor() as cursor:
cursor.execute(f"SAVEPOINT {name}")
self.savepoints.append(name)
return name
def rollback_to_savepoint(self, name: str):
"""回滚到指定保存点"""
with self.connection.cursor() as cursor:
cursor.execute(f"ROLLBACK TO SAVEPOINT {name}")
def release_savepoint(self, name: str):
"""释放保存点"""
with self.connection.cursor() as cursor:
cursor.execute(f"RELEASE SAVEPOINT {name}")
if name in self.savepoints:
self.savepoints.remove(name)
# 使用示例
def advanced_transaction_example():
"""高级事务管理示例"""
conn = psycopg2.connect(
host="localhost",
database="testdb",
user="postgres",
password="password"
)
try:
# 使用默认隔离级别
with TransactionManager(conn) as tx:
with conn.cursor() as cursor:
# 批量插入产品
products = [
('平板电脑', 3999.99, 60),
('蓝牙耳机', 599.99, 150),
('充电宝', 199.99, 300)
]
for product in products:
cursor.execute("""
INSERT INTO products (name, price, stock)
VALUES (%s, %s, %s)
""", product)
# 创建保存点
sp1 = tx.savepoint("inventory_update")
try:
# 更新库存（可能失败）
cursor.execute("""
UPDATE products
SET stock = stock + 10
WHERE name = '笔记本电脑'
""")
print("库存更新成功")
except Exception as e:
print(f"库存更新失败: {e}")
tx.rollback_to_savepoint(sp1)
print("回滚库存更新")
# 继续其他操作
cursor.execute("SELECT COUNT(*) FROM products")
total_products = cursor.fetchone()[0]
print(f"总产品数: {total_products}")
except Exception as e:
print(f"高级事务示例失败: {e}")
finally:
conn.close()
---
07.性能优化
a.批量操作
在事务中执行批量操作可以显著提高性能，减少网络往返和事务开销。使用executemany()方法批量执行SQL语句。
b.预编译语句
使用预编译语句可以提高SQL执行效率，减少SQL解析开销。psycopg2支持参数化查询和语句缓存。
c.连接复用
合理复用数据库连接，避免频繁创建和销毁连接带来的性能开销。使用连接池管理连接生命周期。
4.3 嵌套事务
01.嵌套事务概念
a.定义与特点
PostgreSQL本身不支持真正的嵌套事务，但通过保存点机制可以实现类似功能。嵌套事务允许在事务内部创建子事务，实现部分回滚而不影响整个外层事务。
b.应用场景
适用于复杂的业务逻辑处理，需要将大事务分解为多个小事务的场景。例如订单处理中包含多个步骤，某些步骤失败时只回滚失败部分，不影响其他步骤。
c.实现原理
基于PostgreSQL的保存点功能，每个嵌套层对应一个保存点。回滚嵌套事务时回滚到对应保存点，提交时按顺序释放保存点。
02.保存点机制
a.创建保存点
使用SAVEPOINT命令创建命名保存点，作为嵌套事务的起点。保存点名称在事务内必须唯一，通常使用递增的数字或描述性名称。
b.回滚到保存点
ROLLBACK TO SAVEPOINT命令回滚到指定保存点，撤销该保存点之后的所有操作。保存点之前的操作保持不变，事务继续执行。
c.释放保存点
RELEASE SAVEPOINT命令释放不再需要的保存点，释放相关资源。释放后不能再回滚到该保存点，但可以创建同名的新的保存点。
03.嵌套事务管理
a.事务栈管理
维护一个事务栈，记录当前的嵌套层级和对应的保存点信息。进入嵌套事务时创建保存点并压栈，退出时弹出并处理。
b.异常处理
在嵌套事务中，异常处理更加复杂。需要区分是当前嵌套层的异常还是外层异常，决定是回滚到当前保存点还是回滚整个事务。
c.资源清理
嵌套事务结束时需要正确清理相关资源，包括保存点、临时表、游标等。确保不会出现资源泄漏。
04.实现示例
a.嵌套事务上下文管理器
---
# 嵌套事务上下文管理器示例
import psycopg2
from contextlib import contextmanager
from typing import Optional
class NestedTransactionManager:
def __init__(self, connection):
self.connection = connection
self.savepoint_stack = []
self.savepoint_counter = 0
def begin_nested(self):
"""开始嵌套事务"""
self.savepoint_counter += 1
savepoint_name = f"sp_{self.savepoint_counter}"
self.connection.execute(f"SAVEPOINT {savepoint_name}")
self.savepoint_stack.append(savepoint_name)
return savepoint_name
def rollback_nested(self, savepoint_name: Optional[str] = None):
"""回滚嵌套事务"""
if savepoint_name is None and self.savepoint_stack:
savepoint_name = self.savepoint_stack.pop()
elif savepoint_name in self.savepoint_stack:
# 移除该保存点及之后的所有保存点
index = self.savepoint_stack.index(savepoint_name)
self.savepoint_stack = self.savepoint_stack[:index]
if savepoint_name:
self.connection.execute(f"ROLLBACK TO SAVEPOINT {savepoint_name}")
def commit_nested(self, savepoint_name: Optional[str] = None):
"""提交嵌套事务"""
if savepoint_name is None and self.savepoint_stack:
savepoint_name = self.savepoint_stack.pop()
if savepoint_name:
self.connection.execute(f"RELEASE SAVEPOINT {savepoint_name}")
@contextmanager
def nested_transaction(self):
"""嵌套事务上下文管理器"""
savepoint_name = self.begin_nested()
try:
yield savepoint_name
self.commit_nested(savepoint_name)
except Exception:
self.rollback_nested(savepoint_name)
raise
---
b.业务逻辑示例
---
# 嵌套事务业务逻辑示例
def complex_order_processing(connection, order_data):
"""复杂订单处理，使用嵌套事务"""
tx_manager = NestedTransactionManager(connection)
try:
# 外层事务开始
with connection.cursor() as cursor:
cursor.execute("BEGIN")
# 第一步：创建订单记录
with connection.cursor() as cursor:
cursor.execute(
"INSERT INTO orders (user_id, total_amount) VALUES (%s, %s) RETURNING id",
(order_data['user_id'], order_data['total_amount'])
)
order_id = cursor.fetchone()[0]
# 第二步：处理订单项（嵌套事务）
with tx_manager.nested_transaction():
for item in order_data['items']:
with connection.cursor() as cursor:
# 检查库存
cursor.execute(
"SELECT stock FROM products WHERE id = %s FOR UPDATE",
(item['product_id'],)
)
stock = cursor.fetchone()[0]
if stock < item['quantity']:
raise ValueError(f"库存不足：商品 {item['product_id']}")
# 扣减库存
cursor.execute(
"UPDATE products SET stock = stock - %s WHERE id = %s",
(item['quantity'], item['product_id'])
)
# 创建订单项
cursor.execute(
"INSERT INTO order_items (order_id, product_id, quantity, price) VALUES (%s, %s, %s, %s)",
(order_id, item['product_id'], item['quantity'], item['price'])
)
# 第三步：更新用户积分（另一个嵌套事务）
with tx_manager.nested_transaction():
points = int(order_data['total_amount'] * 0.1) # 10%积分
with connection.cursor() as cursor:
cursor.execute(
"UPDATE users SET points = points + %s WHERE id = %s",
(points, order_data['user_id'])
)
# 提交整个事务
connection.commit()
return order_id
except Exception as e:
connection.rollback()
raise e
---
05.最佳实践
a.嵌套深度控制
避免过深的嵌套层次，一般建议不超过3-4层。过深的嵌套会使逻辑复杂化，难以调试和维护。
b.命名规范
使用有意义的保存点名称，便于调试和日志记录。可以包含嵌套层级信息和业务描述。
c.性能考虑
嵌套事务会增加资源消耗，特别是在高并发环境下。评估性能影响，必要时进行优化。
4.4 两阶段提交
01.两阶段提交概念
a.定义与原理
两阶段提交是一种分布式事务协议，用于确保跨多个数据库节点的原子性操作。通过协调者管理所有参与者的提交过程，要么全部提交，要么全部回滚，保证分布式系统的一致性。
b.应用场景
适用于需要跨多个数据库或数据源保持一致性的场景，如分布式银行转账、电商订单处理、库存管理系统等。确保多个系统间的数据操作要么全部成功，要么全部失败。
c.PostgreSQL实现
PostgreSQL通过PREPARE TRANSACTION和COMMIT PREPARED命令支持两阶段提交。需要将max_prepared_transactions参数设置为大于0的值来启用预编译事务功能。
02.协议流程
a.第一阶段：准备阶段
协调者向所有参与者发送PREPARE请求，参与者执行事务操作但不提交，将操作结果锁定并记录到持久化存储。如果可以提交则返回YES，否则返回NO。
b.第二阶段：提交阶段
协调者根据第一阶段的响应决定提交或回滚。如果所有参与者都返回YES，则发送COMMIT请求；如果有任何参与者返回NO或超时，则发送ROLLBACK请求。
c.故障处理
参与者在准备阶段后崩溃，恢复后可以查询预编译事务状态并继续完成协议。协调者崩溃时，参与者需要相互协调或等待协调者恢复。
03.PostgreSQL命令
a.PREPARE TRANSACTION
使用PREPARE TRANSACTION 'transaction_id'命令将当前事务转换为预编译事务状态。事务ID在集群内必须唯一，通常包含节点标识和序列号。
b.COMMIT PREPARED
使用COMMIT PREPARED 'transaction_id'命令提交预编译事务。提交后事务的所有操作永久生效，释放所有锁定的资源。
c.ROLLBACK PREPARED
使用ROLLBACK PREPARED 'transaction_id'命令回滚预编译事务。回滚后撤销所有操作，释放锁定的资源，数据库恢复到事务开始前的状态。
04.实现示例
a.协调者实现
---
# PostgreSQL两阶段提交协调者示例
import psycopg2
import uuid
from typing import List, Dict, Tuple
class TwoPhaseCommitCoordinator:
def __init__(self, participants: List[Dict]):
self.participants = participants # [{'name': 'db1', 'conn': conn}]
self.global_tx_id = None
def begin_global_transaction(self) -> str:
"""开始全局事务"""
self.global_tx_id = f"global_tx_{uuid.uuid4().hex[:8]}"
print(f"开始全局事务: {self.global_tx_id}")
return self.global_tx_id
def prepare_phase(self) -> Tuple[bool, List[str]]:
"""第一阶段：准备所有参与者"""
prepared_participants = []
failed_participants = []
for participant in self.participants:
try:
conn = participant['conn']
with conn.cursor() as cursor:
# 开始本地事务
cursor.execute("BEGIN")
# 执行业务逻辑（这里需要根据具体业务实现）
self._execute_business_logic(conn, participant['name'])
# 准备事务
with conn.cursor() as cursor:
cursor.execute(f"PREPARE TRANSACTION '{self.global_tx_id}_{participant['name']}'")
prepared_participants.append(participant['name'])
print(f"参与者 {participant['name']} 准备成功")
except Exception as e:
failed_participants.append(participant['name'])
print(f"参与者 {participant['name']} 准备失败: {e}")
# 回滚失败的参与者
try:
with participant['conn'].cursor() as cursor:
cursor.execute("ROLLBACK")
except:
pass
all_prepared = len(failed_participants) == 0
return all_prepared, prepared_participants
def commit_phase(self, prepared_participants: List[str]) -> bool:
"""第二阶段：提交或回滚"""
success = True
for participant in self.participants:
if participant['name'] not in prepared_participants:
continue
try:
conn = participant['conn']
with conn.cursor() as cursor:
cursor.execute(f"COMMIT PREPARED '{self.global_tx_id}_{participant['name']}'")
print(f"参与者 {participant['name']} 提交成功")
except Exception as e:
print(f"参与者 {participant['name']} 提交失败: {e}")
success = False
return success
def rollback_phase(self, prepared_participants: List[str]):
"""回滚阶段"""
for participant in self.participants:
if participant['name'] not in prepared_participants:
continue
try:
conn = participant['conn']
with conn.cursor() as cursor:
cursor.execute(f"ROLLBACK PREPARED '{self.global_tx_id}_{participant['name']}'")
print(f"参与者 {participant['name']} 回滚成功")
except Exception as e:
print(f"参与者 {participant['name']} 回滚失败: {e}")
def execute_global_transaction(self, business_logic_func) -> bool:
"""执行全局事务"""
try:
# 开始全局事务
tx_id = self.begin_global_transaction()
# 第一阶段：准备
all_prepared, prepared_participants = self.prepare_phase()
if all_prepared:
# 第二阶段：提交
success = self.commit_phase(prepared_participants)
if success:
print(f"全局事务 {tx_id} 提交成功")
return True
else:
print(f"全局事务 {tx_id} 提交失败，执行回滚")
self.rollback_phase(prepared_participants)
return False
else:
print(f"全局事务 {tx_id} 准备失败，执行回滚")
self.rollback_phase(prepared_participants)
return False
except Exception as e:
print(f"全局事务执行异常: {e}")
return False
def _execute_business_logic(self, conn, participant_name: str):
"""执行业务逻辑（示例）"""
# 这里根据具体业务需求实现
# 例如：转账操作、订单处理等
pass
---
b.应用场景示例
---
# 分布式银行转账示例
def distributed_bank_transfer():
"""分布式银行转账示例"""
# 连接两个银行数据库
conn1 = psycopg2.connect("dbname=bank1 user=postgres")
conn2 = psycopg2.connect("dbname=bank2 user=postgres")
participants = [
{'name': 'bank1', 'conn': conn1},
{'name': 'bank2', 'conn': conn2}
]
coordinator = TwoPhaseCommitCoordinator(participants)
# 执行分布式转账事务
def transfer_business_logic(conn, participant_name):
with conn.cursor() as cursor:
if participant_name == 'bank1':
# 从银行1扣款
cursor.execute("UPDATE accounts SET balance = balance - 1000 WHERE id = 1")
cursor.execute("INSERT INTO transactions (account_id, amount, type) VALUES (1, -1000, 'TRANSFER_OUT')")
else:
# 向银行2存款
cursor.execute("UPDATE accounts SET balance = balance + 1000 WHERE id = 1")
cursor.execute("INSERT INTO transactions (account_id, amount, type) VALUES (1, 1000, 'TRANSFER_IN')")
success = coordinator.execute_global_transaction(transfer_business_logic)
# 清理连接
conn1.close()
conn2.close()
return success
---
05.管理监控
a.预编译事务查询
通过查询pg_prepared_xacts系统视图可以查看当前所有的预编译事务信息，包括事务ID、所有者、数据库状态等。
b.清理过期事务
定期清理长时间未完成的预编译事务，避免资源占用和锁阻塞。可以使用ROLLBACK PREPARED命令强制回滚。
c.性能监控
监控两阶段提交的性能影响，包括锁等待时间、事务完成时间等。优化业务逻辑减少事务持续时间。
4.5 LISTEN与NOTIFY
01.LISTEN与NOTIFY概述
a.基本概念
PostgreSQL的LISTEN和NOTIFY命令提供了一种轻量级的发布/订阅机制，允许数据库会话之间进行异步通信。NOTIFY发送通知消息，LISTEN接收特定频道的通知消息。
b.通信机制
基于频道名称进行消息传递，支持多对多的通信模式。发送方使用NOTIFY命令发送消息到指定频道，所有监听该频道的会话都会收到通知。
c.事务特性
NOTIFY消息在事务提交时才发送，如果事务回滚则消息不会被发送。这确保了通知与数据库状态的一致性。
02.基础用法
a.LISTEN命令
使用LISTEN channel_name开始监听指定频道的通知消息。一个会话可以同时监听多个频道，也可以多次监听同一频道。
b.NOTIFY命令
使用NOTIFY channel_name [, payload]发送通知消息。payload是可选的消息内容，最大长度为8000字节。
c.UNLISTEN命令
使用UNLISTEN channel_name停止监听指定频道，或使用UNLISTEN *停止监听所有频道。
03.通知机制
a.异步通知
通知是异步传递的，发送方不需要等待接收方处理。接收方需要在查询时检查是否有待处理的通知消息。
b.消息队列
PostgreSQL为每个连接维护一个通知队列，如果接收方没有及时处理消息，消息会在队列中等待。
c.消息格式
通知消息包含发送进程ID、频道名称和载荷内容三个部分，可以通过psycopg2获取这些信息。
04.应用场景
a.缓存失效
当数据库数据发生变化时，通过NOTIFY通知应用层缓存失效，实现数据一致性。适用于高频读取、低频写入的场景。
b.事件驱动架构
数据库事件触发器发送通知，应用层接收通知后执行相应的业务逻辑处理。实现松耦合的事件驱动系统。
c.实时通知系统
多用户系统中，一个用户的操作会通知其他相关用户，如聊天消息、状态更新等实时通知功能。
05.实现示例
a.基础通知处理
---
# PostgreSQL LISTEN/NOTIFY基础示例
import psycopg2
import psycopg2.extensions
import select
import threading
import time
class NotificationListener:
def __init__(self, connection_params):
self.connection_params = connection_params
self.connection = None
self.listening = False
def connect(self):
"""建立数据库连接并设置异步通知"""
self.connection = psycopg2.connect(**self.connection_params)
self.connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
def listen(self, channel_name):
"""开始监听指定频道"""
if not self.connection:
self.connect()
with self.connection.cursor() as cursor:
cursor.execute(f"LISTEN {channel_name}")
self.listening = True
print(f"开始监听频道: {channel_name}")
def unlisten(self, channel_name=None):
"""停止监听频道"""
if not self.connection:
return
with self.connection.cursor() as cursor:
if channel_name:
cursor.execute(f"UNLISTEN {channel_name}")
print(f"停止监听频道: {channel_name}")
else:
cursor.execute("UNLISTEN *")
print("停止监听所有频道")
self.listening = False
def wait_for_notifications(self, timeout=30):
"""等待通知消息"""
if not self.listening:
print("未开始监听任何频道")
return
# 等待连接可读
if select.select([self.connection], [], [], timeout) == ([], [], []):
print(f"等待{timeout}秒后超时")
return
# 处理通知
self.connection.poll()
while self.connection.notifies:
notify = self.connection.notifies.pop(0)
self._handle_notification(notify)
def _handle_notification(self, notification):
"""处理接收到的通知"""
print(f"收到通知: 频道={notification.channel}, "
f"PID={notification.pid}, "
f"消息={notification.payload}")
def close(self):
"""关闭连接"""
if self.connection:
self.unlisten()
self.connection.close()
self.connection = None
# 使用示例
def notification_example():
"""通知使用示例"""
db_params = {
'host': 'localhost',
'database': 'testdb',
'user': 'postgres',
'password': 'password'
}
# 创建通知监听器
listener = NotificationListener(db_params)
listener.listen('order_updates')
try:
print("等待通知消息...")
while True:
listener.wait_for_notifications(timeout=10)
time.sleep(1) # 避免CPU占用过高
except KeyboardInterrupt:
print("停止监听")
finally:
listener.close()
---
b.高级通知管理
---
# 高级通知管理器
import json
from typing import Callable, Dict, List
class NotificationManager:
def __init__(self, connection_params):
self.connection_params = connection_params
self.listeners: Dict[str, List[Callable]] = {}
self.running = False
def add_listener(self, channel: str, callback: Callable):
"""添加频道监听器"""
if channel not in self.listeners:
self.listeners[channel] = []
self.listeners[channel].append(callback)
print(f"添加监听器: 频道={channel}")
def remove_listener(self, channel: str, callback: Callable = None):
"""移除频道监听器"""
if channel in self.listeners:
if callback:
self.listeners[channel].remove(callback)
else:
self.listeners[channel].clear()
print(f"移除监听器: 频道={channel}")
def start_listening(self):
"""开始监听所有频道"""
self.running = True
self.listener_thread = threading.Thread(target=self._listen_loop)
self.listener_thread.daemon = True
self.listener_thread.start()
print("通知管理器已启动")
def stop_listening(self):
"""停止监听"""
self.running = False
if hasattr(self, 'listener_thread'):
self.listener_thread.join()
print("通知管理器已停止")
def _listen_loop(self):
"""监听循环"""
connection = psycopg2.connect(**self.connection_params)
connection.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
try:
# 监听所有频道
with connection.cursor() as cursor:
for channel in self.listeners.keys():
cursor.execute(f"LISTEN {channel}")
while self.running:
if select.select([connection], [], [], 1) == ([], [], []):
continue
connection.poll()
while connection.notifies:
notify = connection.notifies.pop(0)
self._dispatch_notification(notify)
finally:
connection.close()
def _dispatch_notification(self, notify):
"""分发通知消息"""
channel = notify.channel
payload = notify.payload
if channel in self.listeners:
try:
# 解析JSON格式的载荷
if payload:
data = json.loads(payload)
else:
data = {}
# 调用所有监听器
for callback in self.listeners[channel]:
try:
callback(channel, data, notify.pid)
except Exception as e:
print(f"回调函数执行失败: {e}")
except json.JSONDecodeError:
print(f"无法解析通知载荷: {payload}")
def send_notification(self, channel: str, data: dict):
"""发送通知消息"""
connection = psycopg2.connect(**self.connection_params)
try:
with connection.cursor() as cursor:
payload = json.dumps(data)
cursor.execute("NOTIFY %s, %s", (channel, payload))
print(f"发送通知: 频道={channel}, 数据={data}")
finally:
connection.close()
---
06.性能考虑
a.连接资源
每个监听器需要保持一个数据库连接，在大量监听器的情况下需要考虑连接池管理和资源限制。
b.消息处理效率
避免在通知处理中执行耗时操作，考虑使用队列机制异步处理业务逻辑。
c.网络延迟
通知传递存在网络延迟，对于实时性要求高的场景需要评估延迟影响。
5. ORM事务
5.1 SQLAlchemy事务
01.SQLAlchemy事务概述
a.核心概念
SQLAlchemy提供了强大的事务管理机制，支持自动提交模式和显式事务控制。通过Session对象管理数据库操作的生命周期，确保操作的一致性和完整性。
b.事务模式
支持两种事务模式：自动提交模式（autocommit=True）和手动提交模式（默认）。手动模式下，需要显式调用commit()或rollback()来控制事务。
c.ORM集成
与SQLAlchemy ORM深度集成，对象状态的变更会自动在事务中处理。包括对象的增删改查都会被事务管理器跟踪和控制。
02.Session事务管理
a.基本操作
使用session.begin()开始事务，session.commit()提交事务，session.rollback()回滚事务。Session对象会自动跟踪对象的状态变化。
b.上下文管理
推荐使用with语句管理Session生命周期，确保事务的正确提交或回滚。上下文管理器会自动处理异常情况。
c.自动刷新
Session支持自动刷新模式，对象状态变化会自动同步到数据库。也可以禁用自动刷新，手动控制刷新时机。
03.事务隔离级别
a.隔离级别设置
支持设置数据库标准隔离级别：READ UNCOMMITTED、READ COMMITTED、REPEATABLE READ、SERIALIZABLE。
b.引擎级别配置
可以在创建引擎时设置默认隔离级别，也可以在特定事务中覆盖默认设置。
c.死锁处理
提供死锁检测和自动重试机制，当遇到死锁异常时可以自动重新执行事务。
04.嵌套事务
a.savepoint机制
支持嵌套事务通过savepoint实现，可以在大事务中创建子事务，实现部分回滚功能。
b.事务嵌套
使用session.begin_nested()创建嵌套事务，支持多层嵌套结构，每层都可以独立回滚。
c.异常处理
嵌套事务中的异常处理更加复杂，需要明确回滚范围，避免影响父事务的状态。
05.连接池管理
a.连接池配置
SQLAlchemy提供了强大的连接池管理功能，支持多种连接池策略和参数配置。
b.事务隔离
每个Session从连接池获取独立连接，确保事务之间不会相互干扰。
c.连接回收
事务结束后连接会自动回收，避免连接泄漏和资源浪费。
06.实现示例
a.基础事务管理
---
# SQLAlchemy基础事务管理示例
from sqlalchemy import create_engine, Column, Integer, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import sessionmaker, Session
from contextlib import contextmanager
Base = declarative_base()
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
name = Column(String(50))
email = Column(String(100))
# 创建引擎和Session工厂
engine = create_engine('postgresql://user:password@localhost/testdb')
SessionLocal = sessionmaker(bind=engine)
@contextmanager
def get_session():
"""Session上下文管理器"""
session = SessionLocal()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
def create_user_with_transaction(name: str, email: str):
"""在事务中创建用户"""
with get_session() as session:
user = User(name=name, email=email)
session.add(user)
session.flush() # 获取生成的ID
print(f"创建用户成功，ID: {user.id}")
return user
def batch_create_users(users_data):
"""批量创建用户，使用事务"""
with get_session() as session:
users = []
for user_data in users_data:
user = User(**user_data)
users.append(user)
session.add(user)
session.flush() # 批量刷新，获取所有ID
user_ids = [user.id for user in users]
print(f"批量创建用户成功，IDs: {user_ids}")
return user_ids
---
b.嵌套事务处理
---
# 嵌套事务和保存点示例
def complex_user_operation(user_id: int):
"""复杂用户操作，使用嵌套事务"""
with get_session() as session:
# 获取用户
user = session.query(User).filter(User.id == user_id).first()
if not user:
raise ValueError("用户不存在")
# 开始嵌套事务更新用户信息
with session.begin_nested():
user.name = f"Updated_{user.name}"
session.add(user)
# 模拟可能的失败情况
if len(user.name) > 100:
raise ValueError("用户名过长")
# 嵌套事务自动提交
# 另一个嵌套事务处理相关操作
with session.begin_nested():
user.email = f"{user.name.lower()}@example.com"
session.add(user)
# 嵌套事务自动提交
return user
---
c.事务装饰器
---
# 事务装饰器示例
from functools import wraps
from typing import Callable, Any
def transactional(session_factory: Callable = SessionLocal):
"""事务装饰器"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs) -> Any:
session = session_factory()
try:
result = func(session, *args, **kwargs)
session.commit()
return result
except Exception as e:
session.rollback()
raise e
finally:
session.close()
return wrapper
return decorator
# 使用装饰器
@transactional()
def update_user_profile(session: Session, user_id: int, name: str = None, email: str = None):
"""使用装饰器的事务函数"""
user = session.query(User).filter(User.id == user_id).first()
if not user:
raise ValueError("用户不存在")
if name:
user.name = name
if email:
user.email = email
session.add(user)
return user
---
07.性能优化
a.批量操作
使用session.bulk_insert_mappings()和session.bulk_update_mappings()进行批量操作，提高性能。
b.查询优化
合理使用session.flush()控制写入时机，减少数据库往返次数。
c.缓存策略
利用SQLAlchemy的二级缓存和查询缓存机制，提高数据访问效率。
5.2 Django ORM事务
01.Django事务概述
a.核心机制
Django提供了强大的事务管理功能，通过原子性操作确保数据库操作的完整性。Django默认采用自动提交模式，每个查询都会立即执行，但可以通过装饰器或上下文管理器控制事务行为。
b.事务模式
支持两种事务模式：ATOMIC_REQUESTS全局自动事务模式和手动事务控制。全局模式下每个HTTP请求都会在事务中执行，确保请求期间的操作要么全部成功要么全部失败。
c.数据库兼容性
Django ORM事务处理兼容所有支持的数据库，包括PostgreSQL、MySQL、SQLite等，根据数据库特性提供相应的事务支持。
02.atomic装饰器
a.基础用法
使用@atomic装饰器或atomic()上下文管理器创建原子性代码块，确保块内操作要么全部成功要么全部失败。装饰器可以应用于函数或方法，提供简洁的事务控制。
b.嵌套事务
支持通过savepoints实现嵌套事务，内层事务失败不会影响外层事务。可以在atomic块内创建子事务，实现部分回滚功能。
c.异常处理
原子块内发生异常时自动回滚，可以通过try-except捕获异常并处理回滚逻辑。支持设置异常时不回滚的safe_transactions选项。
03.事务控制
a.手动控制
使用transaction.atomic()上下文管理器进行细粒度的事务控制，可以精确控制事务的开始、提交和回滚时机。
b.savepoint操作
在事务内部创建保存点，实现部分回滚功能。可以使用transaction.savepoint()、transaction.savepoint_commit()和transaction.savepoint_rollback()操作保存点。
c.事务隔离级别
支持设置数据库事务隔离级别，可以通过set_isolation_level()方法控制事务的隔离行为。
04.钩子函数
a.事务前钩子
使用transaction.on_commit()注册在事务成功提交后执行的回调函数，确保操作只在事务成功时执行。
b.事务后钩子
提供事务完成后的回调机制，可以用于发送通知、更新缓存等后续操作。支持多个回调函数的注册和执行。
c.错误处理钩子
事务失败时的回调处理机制，可以用于清理资源、记录日志等错误恢复操作。
05.实现示例
a.基础事务操作
---
# Django ORM事务基础操作示例
from django.db import transaction, models
from django.db.models import F, Sum
from django.core.exceptions import ValidationError
from functools import wraps
class Account(models.Model):
"""账户模型"""
name = models.CharField(max_length=100)
balance = models.DecimalField(max_digits=15, decimal_places=2)
created_at = models.DateTimeField(auto_now_add=True)
class Meta:
app_label = 'banking'
def __str__(self):
return f"{self.name}: {self.balance}"
class Transaction(models.Model):
"""交易记录模型"""
from_account = models.ForeignKey(Account, on_delete=models.CASCADE, related_name='outgoing_transactions')
to_account = models.ForeignKey(Account, on_delete=models.CASCADE, related_name='incoming_transactions')
amount = models.DecimalField(max_digits=15, decimal_places=2)
timestamp = models.DateTimeField(auto_now_add=True)
status = models.CharField(max_length=20, default='pending')
class Meta:
app_label = 'banking'
def atomic_transfer_example():
"""原子性转账示例"""
try:
with transaction.atomic():
# 获取账户
from_account = Account.objects.select_for_update().get(name='张三')
to_account = Account.objects.select_for_update().get(name='李四')
transfer_amount = 1000.00
# 检查余额
if from_account.balance < transfer_amount:
raise ValueError("余额不足")
# 扣款
from_account.balance = F('balance') - transfer_amount
from_account.save()
# 存款
to_account.balance = F('balance') + transfer_amount
to_account.save()
# 创建交易记录
transaction_record = Transaction.objects.create(
from_account=from_account,
to_account=to_account,
amount=transfer_amount,
status='completed'
)
print(f"转账成功: {transfer_amount} 从 {from_account.name} 到 {to_account.name}")
return transaction_record
except ValueError as e:
print(f"转账失败: {e}")
return None
except Exception as e:
print(f"转账异常: {e}")
return None
@transaction.atomic
def decorated_transfer(from_name: str, to_name: str, amount: float):
"""使用装饰器的转账函数"""
try:
from_account = Account.objects.select_for_update().get(name=from_name)
to_account = Account.objects.select_for_update().get(name=to_name)
if from_account.balance < amount:
raise ValidationError("余额不足")
from_account.balance -= amount
to_account.balance += amount
from_account.save()
to_account.save()
Transaction.objects.create(
from_account=from_account,
to_account=to_account,
amount=amount,
status='completed'
)
return True
except Exception as e:
print(f"装饰器转账失败: {e}")
return False
---
b.嵌套事务处理
---
# Django嵌套事务示例
def complex_business_operation():
"""复杂业务操作，使用嵌套事务"""
with transaction.atomic(): # 外层事务
print("开始外层事务")
try:
# 创建订单
order = create_order()
print(f"创建订单: {order.id}")
with transaction.atomic(): # 内层事务1
print("开始内层事务1: 库存扣减")
try:
# 扣减库存
reduce_inventory(order.items)
print("库存扣减成功")
except Exception as e:
print(f"库存扣减失败: {e}")
# 内层事务回滚，不影响外层
raise
with transaction.atomic(): # 内层事务2
print("开始内层事务2: 优惠券使用")
try:
# 使用优惠券
apply_coupon(order)
print("优惠券使用成功")
except Exception as e:
print(f"优惠券使用失败: {e}")
# 内层事务回滚，继续执行
pass
# 订单完成
order.status = 'completed'
order.save()
print("订单完成")
except Exception as e:
print(f"外层事务失败: {e}")
raise # 外层事务回滚
def create_order():
"""创建订单"""
# 模拟订单创建逻辑
from django.db import models
class Order(models.Model):
status = models.CharField(max_length=20, default='pending')
total_amount = models.DecimalField(max_digits=10, decimal_places=2)
class Meta:
app_label = 'banking'
order = Order.objects.create(total_amount=100.00)
return order
def reduce_inventory(items):
"""扣减库存"""
if not items: # 模拟失败情况
raise Exception("库存不足")
print(f"扣减库存: {items}")
def apply_coupon(order):
"""应用优惠券"""
# 模拟优惠券使用失败
if order.total_amount < 50:
raise Exception("优惠券不可用")
print(f"订单 {order.id} 使用优惠券")
---
c.事务钩子使用
---
# Django事务钩子示例
from django.core.mail import send_mail
from django.contrib.sessions.models import Session
import logging
logger = logging.getLogger(__name__)
def transaction_with_hooks():
"""使用事务钩子的示例"""
try:
with transaction.atomic():
# 执行数据库操作
account = Account.objects.get(name='王五')
account.balance += 500
account.save()
# 注册事务提交后执行的钩子
transaction.on_commit(
lambda: send_balance_notification(account.id, account.balance)
)
# 注册多个钩子
transaction.on_commit(
lambda: log_transaction_success(account.id, '存款', 500)
)
# 注册缓存清理钩子
transaction.on_commit(
lambda: clear_user_cache(account.name)
)
print("数据库操作完成，等待事务提交...")
except Exception as e:
print(f"事务失败: {e}")
# 钩子函数不会执行
def send_balance_notification(user_id, new_balance):
"""发送余额通知"""
try:
send_mail(
subject='余额变动通知',
message=f'您的账户余额已更新为: {new_balance}',
from_email='[email protected]',
recipient_list=['[email protected]'],
)
print(f"发送余额通知给用户 {user_id}")
except Exception as e:
logger.error(f"发送邮件失败: {e}")
def log_transaction_success(user_id, transaction_type, amount):
"""记录交易成功日志"""
logger.info(f"用户 {user_id} {transaction_type} {amount} 成功")
def clear_user_cache(username):
"""清理用户缓存"""
try:
# 模拟缓存清理
Session.objects.filter(session_key__contains=username).delete()
print(f"清理用户 {username} 的缓存")
except Exception as e:
logger.error(f"清理缓存失败: {e}")
def batch_transaction_with_hooks():
"""批量事务操作与钩子"""
operations = []
try:
with transaction.atomic():
for i in range(3):
account = Account.objects.create(
name=f'批量用户{i}',
balance=1000.00
)
operations.append(('create', account.id))
# 为每个操作注册钩子
transaction.on_commit(
lambda aid=account.id: process_new_account(aid)
)
print(f"批量创建 {len(operations)} 个账户")
except Exception as e:
print(f"批量操作失败: {e}")
def process_new_account(account_id):
"""处理新账户"""
print(f"处理新账户 {account_id}")
# 发送欢迎邮件、初始化设置等
---
06.错误处理与恢复
a.死锁处理
使用select_for_update()时可能遇到死锁，Django提供自动重试机制。可以通过nowait=True设置非等待模式，或skip_locked=True跳过锁定的行。
b.重试机制
实现智能重试机制，处理暂时性数据库错误。使用装饰器模式封装重试逻辑，支持指数退避算法。
c.数据一致性保证
通过select_for_update()锁定记录，确保并发操作的数据一致性。结合乐观锁和悲观锁策略处理并发冲突。
07.性能优化
a.批量操作
使用bulk_create()、bulk_update()进行批量数据库操作，减少查询次数。在事务中执行批量操作提高性能。
b.查询优化
使用select_related()、prefetch_related()减少数据库查询次数。在事务外进行查询准备，事务内只执行必要的写操作。
c.连接管理
合理控制事务生命周期，避免长时间占用数据库连接。使用连接池管理数据库连接资源。
5.3 session管理
01.Session基础概念
a.Session定义
数据库Session是应用程序与数据库交互的核心接口，提供了连接管理、事务控制、状态跟踪等功能。Session作为工作单元，封装了数据库连接和操作的上下文环境。
b.生命周期管理
Session从创建到关闭经历完整生命周期，包括初始化、活跃状态、事务提交/回滚、资源清理等阶段。合理的生命周期管理对于系统性能和资源利用至关重要。
c.线程安全性
Session通常是线程不安全的，每个线程应该使用独立的Session实例。在线程池环境下需要特别注意Session的创建和销毁，避免并发冲突。
02.SQLAlchemy Session
a.Session工厂模式
使用sessionmaker创建Session工厂，提供标准化的Session创建方式。工厂模式确保所有Session使用相同的配置参数，简化Session管理复杂性。
b.上下文管理
推荐使用with语句管理Session生命周期，自动处理commit/rollback和close操作。上下文管理器确保异常情况下的资源正确释放。
c.作用域管理
SQLAlchemy提供scoped_session实现线程局部存储，确保每个线程获取到独立的Session实例。适用于Web应用等多线程环境。
d.实现示例
---
# SQLAlchemy Session管理示例
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker, scoped_session
from contextlib import contextmanager
from typing import Optional, Any
import threading
# 创建引擎
engine = create_engine('postgresql://user:password@localhost/testdb')
# Session工厂
SessionFactory = sessionmaker(bind=engine)
# 线程局部Session
ScopedSession = scoped_session(SessionFactory)
class SessionManager:
"""Session管理器"""
def __init__(self):
self._local = threading.local()
@contextmanager
def get_session(self, scoped=False):
"""获取Session上下文管理器"""
if scoped:
session = ScopedSession()
else:
session = SessionFactory()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
if not scoped:
session.close()
else:
ScopedSession.remove()
def get_current_session(self) -> Optional[Any]:
"""获取当前线程的Session"""
return getattr(self._local, 'session', None)
def set_current_session(self, session):
"""设置当前线程的Session"""
self._local.session = session
def remove_current_session(self):
"""移除当前线程的Session"""
if hasattr(self._local, 'session'):
self._local.session.close()
delattr(self._local, 'session')
# 全局Session管理器实例
session_manager = SessionManager()
def basic_session_operations():
"""基础Session操作示例"""
with session_manager.get_session() as session:
# 执行数据库操作
from models import User
users = session.query(User).filter(User.active == True).all()
print(f"活跃用户数量: {len(users)}")
# 创建新用户
new_user = User(name="新用户", email="[email protected]")
session.add(new_user)
# 自动commit
def scoped_session_example():
"""作用域Session示例"""
# 在线程中使用
def worker_function(worker_id):
session = ScopedSession()
try:
# 执行业务操作
user = session.query(User).get(1)
print(f"Worker {worker_id} 处理用户: {user.name}")
session.commit()
finally:
ScopedSession.remove()
---
03.Django Session管理
a.数据库Session
Django提供基于数据库的Session存储，使用django_session表存储Session数据。支持跨多个应用服务器的Session共享，适用于分布式部署。
b.缓存Session
使用Redis、Memcached等缓存系统存储Session，提高访问性能。缓存Session支持过期策略，但需要注意缓存失效时的Session丢失问题。
c.文件Session
基于文件系统的Session存储，适用于单服务器部署。配置简单，但在高并发环境下可能存在性能瓶颈。
d.Session配置
---
# Django Session配置示例
# settings.py
# 数据库Session配置
SESSION_ENGINE = 'django.contrib.sessions.backends.db'
SESSION_COOKIE_AGE = 86400 # 24小时
SESSION_COOKIE_DOMAIN = None
SESSION_COOKIE_SECURE = False
SESSION_COOKIE_HTTPONLY = True
SESSION_SAVE_EVERY_REQUEST = False
# 缓存Session配置
# CACHES = {
# 'default': {
# 'BACKEND': 'django.core.cache.backends.redis.RedisCache',
# 'LOCATION': 'redis://127.0.0.1:6379/1',
# 'OPTIONS': {
# 'CLIENT_CLASS': 'django_redis.client.DefaultClient',
# }
# }
# }
# SESSION_ENGINE = 'django.contrib.sessions.backends.cache'
# 自定义Session中间件
from django.contrib.sessions.middleware import SessionMiddleware
from django.utils.functional import SimpleLazyObject
class CustomSessionMiddleware(SessionMiddleware):
"""自定义Session中间件"""
def process_request(self, request):
session_key = request.COOKIES.get(settings.SESSION_COOKIE_NAME)
request.session = self.SessionStore(session_key)
# 添加Session统计信息
request.session_data = {
'access_count': request.session.get('access_count', 0) + 1,
'last_access': timezone.now().isoformat()
}
def process_response(self, request, response):
# 在响应中设置Session
if hasattr(request, 'session'):
if request.session.modified:
self.SessionStore.save(request.session.session_key,
request.session._session,
request.session.get_expiry_age())
response.set_cookie(
settings.SESSION_COOKIE_NAME,
request.session.session_key,
max_age=request.session.get_expiry_age(),
domain=settings.SESSION_COOKIE_DOMAIN,
path=settings.SESSION_COOKIE_PATH,
secure=settings.SESSION_COOKIE_SECURE,
httponly=settings.SESSION_COOKIE_HTTPONLY,
samesite=settings.SESSION_COOKIE_SAMESITE,
)
return response
# Session工具类
from django.contrib.sessions.models import Session
from django.utils import timezone
class SessionUtils:
"""Session工具类"""
@staticmethod
def get_active_sessions():
"""获取活跃Session"""
return Session.objects.filter(
expire_date__gt=timezone.now()
).count()
@staticmethod
def cleanup_expired_sessions():
"""清理过期Session"""
deleted_count, _ = Session.objects.filter(
expire_date__lte=timezone.now()
).delete()
return deleted_count
@staticmethod
def invalidate_user_sessions(user_id):
"""使指定用户的所有Session失效"""
from django.contrib.auth.models import User
user = User.objects.get(id=user_id)
# 查找该用户的所有Session
sessions = Session.objects.filter(
session_data__contains=f'_auth_user_id:{user_id}'
)
count = sessions.count()
sessions.delete()
return count
@staticmethod
def get_session_user_count():
"""统计在线用户数"""
from django.contrib.auth.models import User
# 获取所有活跃Session
active_sessions = Session.objects.filter(
expire_date__gt=timezone.now()
)
# 解析Session获取用户ID
user_ids = set()
for session in active_sessions:
try:
session_data = session.get_decoded()
if '_auth_user_id' in session_data:
user_ids.add(session_data['_auth_user_id'])
except:
continue
return len(user_ids)
---
04.连接池管理
a.SQLAlchemy连接池
SQLAlchemy内置多种连接池实现，包括QueuePool、StaticPool、NullPool等。连接池配置需要考虑并发连接数、超时时间、回收策略等参数。
b.Django数据库连接
Django使用数据库连接池，每个请求周期内复用数据库连接。配置CONN_MAX_AGE参数控制连接的最大生命周期，平衡连接复用和连接质量。
c.连接池优化
---
# 连接池配置和优化示例
from sqlalchemy import create_engine, event
from sqlalchemy.pool import QueuePool
import logging
# 连接池配置
def create_optimized_engine(database_url):
"""创建优化的数据库引擎"""
engine = create_engine(
database_url,
poolclass=QueuePool,
pool_size=10, # 连接池大小
max_overflow=20, # 最大溢出连接数
pool_pre_ping=True, # 连接前ping检查
pool_recycle=3600, # 连接回收时间(秒)
echo=False, # SQL日志
connect_args={
"connect_timeout": 10,
"application_name": "my_app"
}
)
# 添加连接事件监听
@event.listens_for(engine, "connect")
def receive_connect(dbapi_connection, connection_record):
"""连接建立事件"""
logging.info("数据库连接已建立")
@event.listens_for(engine, "checkout")
def receive_checkout(dbapi_connection, connection_record, connection_proxy):
"""连接检出事件"""
logging.info("从连接池检出连接")
@event.listens_for(engine, "checkin")
def receive_checkin(dbapi_connection, connection_record):
"""连接检入事件"""
logging.info("连接返回连接池")
return engine
# 连接池监控
class ConnectionPoolMonitor:
"""连接池监控器"""
def __init__(self, engine):
self.engine = engine
self.stats = {
'total_connections': 0,
'active_connections': 0,
'pool_hits': 0,
'pool_misses': 0
}
def get_pool_status(self):
"""获取连接池状态"""
pool = self.engine.pool
return {
'size': pool.size(),
'checked_in': pool.checkedin(),
'checked_out': pool.checkedout(),
'overflow': pool.overflow(),
'invalid': pool.invalid()
}
def monitor_connections(self):
"""监控连接使用情况"""
status = self.get_pool_status()
logging.info(f"连接池状态: {status}")
# 检查连接池健康状态
if status['checked_out'] > status['size']:
logging.warning("连接池溢出，考虑增加pool_size")
# Django数据库配置优化
# settings.py
DATABASES = {
'default': {
'ENGINE': 'django.db.backends.postgresql',
'NAME': 'mydatabase',
'USER': 'myuser',
'PASSWORD': 'mypassword',
'HOST': 'localhost',
'PORT': '5432',
'CONN_MAX_AGE': 60, # 连接最大存活时间(秒)
'OPTIONS': {
'connect_timeout': 10,
'application_name': 'django_app',
}
}
}
# 数据库连接中间件
class DatabaseConnectionMiddleware:
"""数据库连接管理中间件"""
def __init__(self, get_response):
self.get_response = get_response
def __call__(self, request):
# 请求开始时的连接检查
from django.db import connection
if connection.connection is None:
logging.info("创建新的数据库连接")
response = self.get_response(request)
# 请求结束时的连接处理
if connection.connection is not None:
# 检查连接是否健康
try:
with connection.cursor() as cursor:
cursor.execute("SELECT 1")
except Exception as e:
logging.error(f"数据库连接异常: {e}")
connection.close()
return response
---
05.Session状态管理
a.ORM状态跟踪
Session跟踪对象的状态变化，包括transient(临时)、pending(待定)、persistent(持久)、detached(分离)四种状态。状态管理确保数据库与对象模型的同步。
b.对象缓存
Session维护对象的第一级缓存，避免重复查询相同对象。缓存管理影响内存使用和查询性能，需要定期清理。
c.状态同步
---
# Session状态管理示例
from sqlalchemy.orm import Session
from sqlalchemy import inspect
from models import User
class SessionStateManager:
"""Session状态管理器"""
def __init__(self, session: Session):
self.session = session
def get_object_states(self):
"""获取Session中所有对象的状态"""
states = {
'transient': [],
'pending': [],
'persistent': [],
'detached': []
}
# 检查Session中的所有对象
for obj in self.session:
state = inspect(obj)
if state.transient:
states['transient'].append(obj)
elif state.pending:
states['pending'].append(obj)
elif state.persistent:
states['persistent'].append(obj)
elif state.detached:
states['detached'].append(obj)
return states
def print_session_info(self):
"""打印Session信息"""
states = self.get_object_states()
print("Session状态信息:")
print(f" 临时对象(transient): {len(states['transient'])}")
print(f" 待定对象(pending): {len(states['pending'])}")
print(f" 持久对象(persistent): {len(states['persistent'])}")
print(f" 分离对象(detached): {len(states['detached'])}")
def sync_with_database(self):
"""与数据库同步"""
self.session.expire_all()
print("已刷新Session中的所有对象")
def clear_session(self):
"""清空Session"""
self.session.expunge_all()
print("已清空Session中的所有对象")
def demonstrate_object_lifecycle():
"""演示对象生命周期"""
session = SessionFactory()
manager = SessionStateManager(session)
try:
# 1. 创建临时对象
user = User(name="测试用户", email="[email protected]")
print(f"创建临时对象: {user.name}")
# 2. 添加到Session，变为pending状态
session.add(user)
manager.print_session_info()
# 3. 提交事务，变为persistent状态
session.commit()
manager.print_session_info()
# 4. 修改持久对象
user.name = "修改后的用户名"
manager.print_session_info()
# 5. 从Session中移除，变为detached状态
session.expunge(user)
manager.print_session_info()
finally:
session.close()
---
06.Session安全与隔离
a.事务隔离
Session提供事务隔离机制，确保并发操作的数据一致性。支持设置隔离级别，包括READ COMMITTED、REPEATABLE READ、SERIALIZABLE等。
b.并发控制
使用乐观锁和悲观锁策略处理并发冲突。乐观锁通过版本号控制，悲观锁通过数据库锁机制控制。
c.Session安全最佳实践
---
# Session安全与并发控制示例
from sqlalchemy import text
from sqlalchemy.orm import with_for_update
from contextlib import contextmanager
import time
class SessionSecurityManager:
"""Session安全管理器"""
def __init__(self, session_factory):
self.session_factory = session_factory
@contextmanager
def isolated_session(self, isolation_level="READ_COMMITTED"):
"""隔离事务上下文"""
session = self.session_factory()
try:
# 设置事务隔离级别
session.execute(
text(f"SET TRANSACTION ISOLATION LEVEL {isolation_level}")
)
session.begin()
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
@contextmanager
def locked_session(self, lock_mode="FOR UPDATE"):
"""锁定事务上下文"""
session = self.session_factory()
try:
session.begin()
yield session, lock_mode
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
# 乐观锁实现
from sqlalchemy import Column, Integer, String, Version
from sqlalchemy.ext.declarative import declarative_base
Base = declarative_base()
class OptimisticUser(Base):
__tablename__ = 'optimistic_users'
id = Column(Integer, primary_key=True)
name = Column(String(100))
version = Column(Integer, default=0) # 版本号字段
def update_with_version_check(self, session, new_name):
"""带版本检查的更新"""
old_version = self.version
self.name = new_name
self.version += 1
# 执行更新，检查版本号
result = session.execute(
text("UPDATE optimistic_users SET name=:name, version=:version "
"WHERE id=:id AND version=:old_version"),
{
'name': new_name,
'version': self.version,
'id': self.id,
'old_version': old_version
}
)
if result.rowcount == 0:
raise Exception("乐观锁冲突: 记录已被其他事务修改")
# 悲观锁使用示例
def pessimistic_lock_example():
"""悲观锁使用示例"""
security_manager = SessionSecurityManager(SessionFactory)
with security_manager.locked_session("FOR UPDATE") as (session, lock_mode):
# 查询并锁定用户记录
user = session.query(User).with_for_update().filter(User.id == 1).first()
if user:
print(f"锁定用户: {user.name}")
# 在锁定状态下进行修改
user.name = "锁定的用户"
time.sleep(5) # 模拟长时间操作
print("用户修改完成")
# 并发测试
import threading
def concurrent_update_test():
"""并发更新测试"""
results = []
def update_user(user_id, new_name, thread_id):
try:
security_manager = SessionSecurityManager(SessionFactory)
with security_manager.isolated_session("SERIALIZABLE"):
session = security_manager.session_factory()
user = session.query(User).filter(User.id == user_id).first()
if user:
print(f"线程{thread_id}开始修改用户: {user.name}")
user.name = f"{new_name}_by_thread_{thread_id}"
time.sleep(1) # 模拟处理时间
results.append(f"线程{thread_id}成功修改为: {user.name}")
session.commit()
session.close()
except Exception as e:
results.append(f"线程{thread_id}失败: {e}")
# 创建多个线程并发修改
threads = []
for i in range(3):
thread = threading.Thread(
target=update_user,
args=(1, "并发用户", i+1)
)
threads.append(thread)
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 输出结果
for result in results:
print(result)
---
07.性能优化策略
a.Session复用
合理复用Session减少连接创建开销，但要注意Session状态积累问题。定期清理或重新创建Session避免内存泄漏。
b.批量操作优化
使用批量插入、更新操作减少数据库交互次数。合理配置批量大小平衡内存使用和执行效率。
c.查询优化
---
# Session性能优化示例
from sqlalchemy.orm import sessionmaker, joinedload, selectinload
from sqlalchemy.sql import text
import time
class PerformanceOptimizedSession:
"""性能优化的Session管理"""
def __init__(self, session_factory):
self.session_factory = session_factory
self.query_stats = {
'total_queries': 0,
'total_time': 0,
'slow_queries': []
}
@contextmanager
def timed_session(self):
"""带时间统计的Session"""
session = self.session_factory()
start_time = time.time()
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
finally:
end_time = time.time()
duration = end_time - start_time
self.query_stats['total_queries'] += 1
self.query_stats['total_time'] += duration
if duration > 1.0: # 超过1秒的查询
self.query_stats['slow_queries'].append({
'duration': duration,
'timestamp': end_time
})
session.close()
def optimized_batch_insert(self, objects, batch_size=1000):
"""优化的批量插入"""
session = self.session_factory()
try:
session.begin()
for i in range(0, len(objects), batch_size):
batch = objects[i:i + batch_size]
session.bulk_save_objects(batch)
session.flush() # 刷新到数据库
print(f"已插入 {i + len(batch)}/{len(objects)} 条记录")
session.commit()
except Exception:
session.rollback()
raise
finally:
session.close()
def optimized_query_with_relations(self):
"""优化的关联查询"""
session = self.session_factory()
try:
# 使用joinedload预加载关联数据
users_with_posts = session.query(User).options(
joinedload(User.posts),
joinedload(User.profile)
).all()
# 使用selectinload处理一对多关系
users_with_comments = session.query(User).options(
selectinload(User.comments)
).all()
return users_with_posts, users_with_comments
finally:
session.close()
def get_performance_stats(self):
"""获取性能统计"""
avg_time = (self.query_stats['total_time'] /
self.query_stats['total_queries']
if self.query_stats['total_queries'] > 0 else 0)
return {
'total_queries': self.query_stats['total_queries'],
'total_time': round(self.query_stats['total_time'], 3),
'average_time': round(avg_time, 3),
'slow_queries_count': len(self.query_stats['slow_queries'])
}
# 使用示例
def performance_optimization_demo():
"""性能优化演示"""
manager = PerformanceOptimizedSession(SessionFactory)
# 1. 使用计时Session
with manager.timed_session() as session:
users = session.query(User).filter(User.active == True).all()
print(f"查询到 {len(users)} 个活跃用户")
# 2. 批量插入优化
new_users = [
User(name=f"用户{i}", email=f"user{i}@example.com")
for i in range(10000)
]
manager.optimized_batch_insert(new_users)
# 3. 关联查询优化
users_with_posts, users_with_comments = manager.optimized_query_with_relations()
# 4. 输出性能统计
stats = manager.get_performance_stats()
print(f"Session性能统计: {stats}")
5.4 事务装饰器
01.装饰器基础概念
a.装饰器定义
装饰器是一种Python设计模式，允许在不修改函数源代码的情况下扩展函数功能。事务装饰器通过包装目标函数，自动处理事务的开始、提交和回滚操作。
b.事务装饰器优势
提供声明式的事务管理，简化代码结构，提高代码可读性和可维护性。通过装饰器模式实现关注点分离，将事务逻辑与业务逻辑解耦。
c.应用场景
适用于需要事务管理的各种业务函数，如数据库操作、服务调用、API处理等。特别适合多层架构中的事务传播和统一管理。
02.基础事务装饰器
a.SQLAlchemy事务装饰器
基于SQLAlchemy Session创建的事务装饰器，自动管理Session生命周期和事务边界。支持配置隔离级别、超时时间等事务参数。
b.Django事务装饰器
Django内置的@transaction.atomic装饰器，提供简单的事务管理功能。支持嵌套事务和保存点机制。
c.基础实现
---
# 基础事务装饰器实现
from functools import wraps
from contextlib import contextmanager
from typing import Callable, Any, Optional
import time
import logging
class TransactionDecorator:
"""事务装饰器基类"""
def __init__(self,
isolation_level: str = "READ_COMMITTED",
timeout: Optional[float] = None,
retry_count: int = 0,
rollback_for: tuple = (Exception,)):
self.isolation_level = isolation_level
self.timeout = timeout
self.retry_count = retry_count
self.rollback_for = rollback_for
def __call__(self, func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs):
return self.execute_with_transaction(func, *args, **kwargs)
return wrapper
def execute_with_transaction(self, func: Callable, *args, **kwargs):
"""执行带事务的函数"""
raise NotImplementedError("子类需要实现此方法")
# SQLAlchemy事务装饰器
class SQLAlchemyTransactionDecorator(TransactionDecorator):
"""SQLAlchemy事务装饰器"""
def __init__(self, session_factory: Callable, **kwargs):
super().__init__(**kwargs)
self.session_factory = session_factory
def execute_with_transaction(self, func: Callable, *args, **kwargs):
session = self.session_factory()
transaction_start = time.time()
try:
# 设置隔离级别
if self.isolation_level:
session.execute(
text(f"SET TRANSACTION ISOLATION LEVEL {self.isolation_level}")
)
session.begin()
# 将session注入到函数参数中
if 'session' in func.__code__.co_varnames:
kwargs['session'] = session
result = func(*args, **kwargs)
session.commit()
return result
except self.rollback_for as e:
session.rollback()
logging.error(f"事务回滚: {func.__name__} - {e}")
raise e
except Exception as e:
session.rollback()
logging.error(f"事务异常回滚: {func.__name__} - {e}")
raise e
finally:
session.close()
# 检查超时
if self.timeout and (time.time() - transaction_start) > self.timeout:
raise TimeoutError(f"事务执行超时: {self.timeout}秒")
# 使用示例
from sqlalchemy.orm import sessionmaker
SessionFactory = sessionmaker(bind=create_engine('postgresql://user:pass@localhost/db'))
@SQLAlchemyTransactionDecorator(SessionFactory, isolation_level="SERIALIZABLE")
def transfer_funds(session, from_account_id: int, to_account_id: int, amount: float):
"""转账函数，使用事务装饰器"""
from models import Account
# 查询账户并锁定
from_account = session.query(Account).filter(
Account.id == from_account_id
).with_for_update().first()
to_account = session.query(Account).filter(
Account.id == to_account_id
).with_for_update().first()
if not from_account or not to_account:
raise ValueError("账户不存在")
if from_account.balance < amount:
raise ValueError("余额不足")
# 执行转账
from_account.balance -= amount
to_account.balance += amount
session.add(from_account)
session.add(to_account)
return f"转账成功: {amount} 从 {from_account.name} 到 {to_account.name}"
---
03.高级事务装饰器
a.重试机制装饰器
集成重试逻辑的事务装饰器，自动处理死锁、超时等可恢复错误。支持指数退避算法和重试次数限制。
b.异步事务装饰器
适配异步函数的事务装饰器，支持async/await语法。适用于异步Web框架和异步数据库操作。
c.高级实现
---
# 高级事务装饰器实现
import asyncio
import random
from dataclasses import dataclass
@dataclass
class RetryConfig:
max_retries: int = 3
base_delay: float = 0.1
max_delay: float = 10.0
backoff_factor: float = 2.0
jitter: bool = True
def calculate_delay(attempt: int, config: RetryConfig) -> float:
"""计算重试延迟"""
delay = config.base_delay * (config.backoff_factor ** attempt)
delay = min(delay, config.max_delay)
if config.jitter:
jitter_range = delay * 0.1
delay += random.uniform(-jitter_range, jitter_range)
return max(0, delay)
class RetryableTransactionDecorator(TransactionDecorator):
"""支持重试的事务装饰器"""
def __init__(self, session_factory: Callable,
retry_config: RetryConfig = None,
retryable_exceptions: tuple = (Exception,), **kwargs):
super().__init__(**kwargs)
self.session_factory = session_factory
self.retry_config = retry_config or RetryConfig()
self.retryable_exceptions = retryable_exceptions
def execute_with_transaction(self, func: Callable, *args, **kwargs):
last_exception = None
for attempt in range(self.retry_config.max_retries + 1):
try:
if attempt > 0:
delay = calculate_delay(attempt - 1, self.retry_config)
logging.info(f"重试第 {attempt} 次，等待 {delay:.2f} 秒")
time.sleep(delay)
session = self.session_factory()
try:
session.begin()
if 'session' in func.__code__.co_varnames:
kwargs['session'] = session
result = func(*args, **kwargs)
session.commit()
if attempt > 0:
logging.info(f"重试成功，总重试次数: {attempt}")
return result
except Exception as e:
session.rollback()
if self._should_retry(e, attempt):
last_exception = e
continue
else:
raise e
finally:
session.close()
except self.retryable_exceptions as e:
if attempt < self.retry_config.max_retries:
last_exception = e
continue
else:
raise e
raise last_exception
def _should_retry(self, exception: Exception, attempt: int) -> bool:
"""判断是否应该重试"""
return (isinstance(exception, self.retryable_exceptions) and
attempt < self.retry_config.max_retries)
# 异步事务装饰器
class AsyncTransactionDecorator(TransactionDecorator):
"""异步事务装饰器"""
def __init__(self, session_factory: Callable, **kwargs):
super().__init__(**kwargs)
self.session_factory = session_factory
async def execute_with_transaction(self, func: Callable, *args, **kwargs):
"""执行异步事务"""
loop = asyncio.get_event_loop()
# 在线程池中执行同步数据库操作
return await loop.run_in_executor(
None,
self._sync_execute_with_transaction,
func, *args, **kwargs
)
def _sync_execute_with_transaction(self, func: Callable, *args, **kwargs):
"""同步执行事务"""
session = self.session_factory()
try:
session.begin()
if 'session' in func.__code__.co_varnames:
kwargs['session'] = session
result = func(*args, **kwargs)
session.commit()
return result
except self.rollback_for as e:
session.rollback()
raise e
finally:
session.close()
def __call__(self, func: Callable) -> Callable:
if asyncio.iscoroutinefunction(func):
@wraps(func)
async def async_wrapper(*args, **kwargs):
return await self.execute_with_transaction(func, *args, **kwargs)
return async_wrapper
else:
@wraps(func)
def sync_wrapper(*args, **kwargs):
return self._sync_execute_with_transaction(func, *args, **kwargs)
return sync_wrapper
# 使用示例
retry_config = RetryConfig(max_retries=5, base_delay=0.5)
@RetryableTransactionDecorator(
SessionFactory,
retry_config=retry_config,
retryable_exceptions=(TimeoutError, ConnectionError)
)
def create_user_with_retry(session, name: str, email: str):
"""带重试的用户创建函数"""
from models import User
# 检查用户是否已存在
existing_user = session.query(User).filter(User.email == email).first()
if existing_user:
raise ValueError(f"邮箱 {email} 已被使用")
user = User(name=name, email=email)
session.add(user)
session.flush() # 获取ID
return user
@AsyncTransactionDecorator(SessionFactory)
async def async_create_user(name: str, email: str):
"""异步创建用户"""
# 模拟异步操作
await asyncio.sleep(0.1)
# 这里只能调用不依赖session的操作
# 实际的数据库操作会在事务装饰器中处理
return f"异步创建用户: {name}, {email}"
---
04.嵌套事务装饰器
a.保存点机制
支持嵌套事务的装饰器，通过保存点实现部分回滚功能。内层事务失败不会影响外层事务的完整性。
b.事务传播行为
定义不同的事务传播行为，如REQUIRED、REQUIRES_NEW、SUPPORTS等。类似于Spring的事务传播机制。
c.传播行为实现
---
# 事务传播行为枚举
from enum import Enum
class PropagationBehavior(Enum):
REQUIRED = "required" # 存在事务则加入，否则创建新事务
REQUIRES_NEW = "requires_new" # 总是创建新事务
SUPPORTS = "supports" # 支持事务，但非必需
NOT_SUPPORTED = "not_supported" # 非事务方式执行
NEVER = "never" # 总是非事务方式执行
MANDATORY = "mandatory" # 必须在事务中执行
class NestedTransactionDecorator(TransactionDecorator):
"""支持嵌套事务的装饰器"""
def __init__(self, session_factory: Callable,
propagation: PropagationBehavior = PropagationBehavior.REQUIRED,
**kwargs):
super().__init__(**kwargs)
self.session_factory = session_factory
self.propagation = propagation
def execute_with_transaction(self, func: Callable, *args, **kwargs):
# 检查当前线程是否已有Session
current_session = self._get_current_session()
if current_session and self.propagation == PropagationBehavior.REQUIRED:
# 加入现有事务
return self._execute_in_existing_transaction(current_session, func, *args, **kwargs)
elif current_session and self.propagation == PropagationBehavior.REQUIRES_NEW:
# 创建新事务（嵌套）
return self._execute_with_savepoint(current_session, func, *args, **kwargs)
elif self.propagation == PropagationBehavior.NOT_SUPPORTED:
# 非事务执行
return self._execute_without_transaction(func, *args, **kwargs)
elif self.propagation == PropagationBehavior.MANDATORY and not current_session:
# 必须有事务
raise RuntimeError("函数必须在事务中执行")
else:
# 创建新事务
return self._execute_with_new_transaction(func, *args, **kwargs)
def _get_current_session(self):
"""获取当前Session"""
# 这里使用线程本地存储实现
import threading
local = threading.local()
return getattr(local, 'current_session', None)
def _set_current_session(self, session):
"""设置当前Session"""
import threading
local = threading.local()
local.current_session = session
def _execute_in_existing_transaction(self, session, func, *args, **kwargs):
"""在现有事务中执行"""
kwargs['session'] = session
return func(*args, **kwargs)
def _execute_with_savepoint(self, session, func, *args, **kwargs):
"""使用保存点执行嵌套事务"""
savepoint = session.begin_nested()
try:
kwargs['session'] = session
result = func(*args, **kwargs)
savepoint.commit()
return result
except Exception:
savepoint.rollback()
raise
def _execute_without_transaction(self, func, *args, **kwargs):
"""非事务执行"""
session = self.session_factory()
try:
kwargs['session'] = session
return func(*args, **kwargs)
finally:
session.close()
def _execute_with_new_transaction(self, func, *args, **kwargs):
"""执行新事务"""
session = self.session_factory()
old_session = self._get_current_session()
try:
self._set_current_session(session)
session.begin()
kwargs['session'] = session
result = func(*args, **kwargs)
session.commit()
return result
except Exception:
session.rollback()
raise
finally:
self._set_current_session(old_session)
session.close()
# 使用示例
@NestedTransactionDecorator(SessionFactory, propagation=PropagationBehavior.REQUIRED)
def update_user_profile(user_id: int, name: str = None, email: str = None, session=None):
"""更新用户档案（必须在外层事务中）"""
from models import User
user = session.query(User).filter(User.id == user_id).first()
if not user:
raise ValueError("用户不存在")
if name:
user.name = name
if email:
user.email = email
session.add(user)
return user
@NestedTransactionDecorator(SessionFactory, propagation=PropagationBehavior.REQUIRES_NEW)
def log_user_activity(user_id: int, activity: str, session=None):
"""记录用户活动（总是新事务）"""
from models import ActivityLog
log = ActivityLog(user_id=user_id, activity=activity)
session.add(log)
return log
def complex_user_operation(user_id: int):
"""复杂用户操作，演示嵌套事务"""
decorator = NestedTransactionDecorator(SessionFactory)
with decorator._execute_with_new_transaction(lambda: None):
session = decorator._get_current_session()
# 外层事务
user = update_user_profile(user_id, name="新名称")
# 内层新事务记录日志
log_user_activity(user_id, "更新用户档案")
# 另一个内层事务
with decorator._execute_with_savepoint(session, lambda: None):
# 更新用户统计信息
pass
---
05.性能监控装饰器
a.执行时间统计
监控事务执行时间，记录慢查询和性能瓶颈。支持设置性能阈值和告警机制。
b.资源使用监控
监控数据库连接、内存使用等资源消耗。帮助识别资源泄漏和性能问题。
c.监控实现
---
# 性能监控装饰器
from typing import Dict, List
import threading
from collections import defaultdict
import psutil
import os
class TransactionMetrics:
"""事务指标收集器"""
def __init__(self):
self.metrics = {
'total_transactions': 0,
'successful_transactions': 0,
'failed_transactions': 0,
'total_time': 0.0,
'slow_transactions': [],
'concurrent_transactions': 0,
'resource_usage': []
}
self.lock = threading.Lock()
def record_transaction(self, duration: float, success: bool, slow_threshold: float = 1.0):
"""记录事务指标"""
with self.lock:
self.metrics['total_transactions'] += 1
self.metrics['total_time'] += duration
if success:
self.metrics['successful_transactions'] += 1
else:
self.metrics['failed_transactions'] += 1
if duration > slow_threshold:
self.metrics['slow_transactions'].append({
'duration': duration,
'timestamp': time.time()
})
def get_stats(self) -> Dict:
"""获取统计信息"""
with self.lock:
total = self.metrics['total_transactions']
if total == 0:
return self.metrics.copy()
avg_time = self.metrics['total_time'] / total
success_rate = self.metrics['successful_transactions'] / total * 100
return {
**self.metrics,
'average_time': avg_time,
'success_rate': success_rate,
'slow_transaction_count': len(self.metrics['slow_transactions'])
}
# 全局指标收集器
global_metrics = TransactionMetrics()
class MonitoredTransactionDecorator(TransactionDecorator):
"""带监控的事务装饰器"""
def __init__(self, session_factory: Callable,
slow_threshold: float = 1.0,
enable_resource_monitoring: bool = False,
**kwargs):
super().__init__(**kwargs)
self.session_factory = session_factory
self.slow_threshold = slow_threshold
self.enable_resource_monitoring = enable_resource_monitoring
def execute_with_transaction(self, func: Callable, *args, **kwargs):
start_time = time.time()
process = psutil.Process(os.getpid())
# 记录开始时的资源使用
if self.enable_resource_monitoring:
start_memory = process.memory_info().rss / 1024 / 1024 # MB
start_cpu = process.cpu_percent()
success = False
try:
session = self.session_factory()
try:
session.begin()
# 增加并发事务计数
with global_metrics.lock:
global_metrics.metrics['concurrent_transactions'] += 1
if 'session' in func.__code__.co_varnames:
kwargs['session'] = session
result = func(*args, **kwargs)
session.commit()
success = True
return result
except Exception as e:
session.rollback()
raise e
finally:
session.close()
# 减少并发事务计数
with global_metrics.lock:
global_metrics.metrics['concurrent_transactions'] -= 1
finally:
duration = time.time() - start_time
# 记录资源使用情况
if self.enable_resource_monitoring:
end_memory = process.memory_info().rss / 1024 / 1024 # MB
end_cpu = process.cpu_percent()
resource_usage = {
'duration': duration,
'memory_delta': end_memory - start_memory,
'cpu_usage': end_cpu
}
global_metrics.metrics['resource_usage'].append(resource_usage)
# 记录事务指标
global_metrics.record_transaction(duration, success, self.slow_threshold)
# 慢事务告警
if duration > self.slow_threshold:
logging.warning(
f"慢事务检测: {func.__name__} 执行时间 {duration:.2f}秒 "
f"(阈值: {self.slow_threshold}秒)"
)
def get_transaction_report() -> Dict:
"""获取事务报告"""
stats = global_metrics.get_stats()
report = {
'事务统计': {
'总事务数': stats['total_transactions'],
'成功事务数': stats['successful_transactions'],
'失败事务数': stats['failed_transactions'],
'成功率': f"{stats['success_rate']:.2f}%",
'平均执行时间': f"{stats['average_time']:.3f}秒"
},
'性能指标': {
'慢事务数量': stats['slow_transaction_count'],
'当前并发事务数': stats['concurrent_transactions']
}
}
if stats['resource_usage']:
avg_memory_delta = sum(r['memory_delta'] for r in stats['resource_usage']) / len(stats['resource_usage'])
avg_cpu_usage = sum(r['cpu_usage'] for r in stats['resource_usage']) / len(stats['resource_usage'])
report['资源使用'] = {
'平均内存变化': f"{avg_memory_delta:.2f}MB",
'平均CPU使用率': f"{avg_cpu_usage:.1f}%"
}
return report
# 使用示例
@MonitoredTransactionDecorator(
SessionFactory,
slow_threshold=2.0,
enable_resource_monitoring=True
)
def complex_business_operation(session):
"""复杂业务操作，带监控"""
from models import Order, Product
# 模拟复杂操作
orders = session.query(Order).filter(Order.status == 'pending').limit(100).all()
for order in orders:
# 更新订单状态
order.status = 'processing'
session.add(order)
# 检查库存
product = session.query(Product).filter(Product.id == order.product_id).first()
if product.stock < order.quantity:
raise ValueError(f"库存不足: {product.name}")
# 扣减库存
product.stock -= order.quantity
session.add(product)
return f"处理了 {len(orders)} 个订单"
# 打印报告
def print_transaction_report():
"""打印事务报告"""
report = get_transaction_report()
print("=== 事务执行报告 ===")
for category, metrics in report.items():
print(f"\n{category}:")
for metric, value in metrics.items():
print(f" {metric}: {value}")
---
06.装饰器组合与最佳实践
a.装饰器链
支持多个装饰器组合使用，实现功能叠加。注意装饰器执行顺序和参数传递。
b.配置管理
统一的配置管理机制，支持从配置文件、环境变量或数据库加载装饰器参数。
c.最佳实践总结
---
# 装饰器组合和最佳实践示例
from typing import Union
class TransactionDecoratorBuilder:
"""事务装饰器构建器"""
def __init__(self, session_factory: Callable):
self.session_factory = session_factory
self.decorators = []
def with_retry(self, max_retries: int = 3, base_delay: float = 0.1):
"""添加重试功能"""
retry_config = RetryConfig(max_retries=max_retries, base_delay=base_delay)
retry_decorator = RetryableTransactionDecorator(
self.session_factory,
retry_config=retry_config
)
self.decorators.append(retry_decorator)
return self
def with_monitoring(self, slow_threshold: float = 1.0):
"""添加监控功能"""
monitor_decorator = MonitoredTransactionDecorator(
self.session_factory,
slow_threshold=slow_threshold
)
self.decorators.append(monitor_decorator)
return self
def with_nested_support(self, propagation: PropagationBehavior = PropagationBehavior.REQUIRED):
"""添加嵌套事务支持"""
nested_decorator = NestedTransactionDecorator(
self.session_factory,
propagation=propagation
)
self.decorators.append(nested_decorator)
return self
def build(self):
"""构建最终装饰器"""
def combined_decorator(func: Callable) -> Callable:
# 按相反顺序应用装饰器（因为装饰器是从内向外执行）
for decorator in reversed(self.decorators):
func = decorator(func)
return func
return combined_decorator
# 使用构建器创建复合装饰器
def create_comprehensive_decorator(session_factory: Callable):
"""创建全面的装饰器"""
return (TransactionDecoratorBuilder(session_factory)
.with_retry(max_retries=3, base_delay=0.5)
.with_monitoring(slow_threshold=2.0)
.with_nested_support(propagation=PropagationBehavior.REQUIRED)
.build())
# 应用复合装饰器
comprehensive_decorator = create_comprehensive_decorator(SessionFactory)
@comprehensive_decorator
def critical_business_operation(session, operation_data: Dict):
"""关键业务操作，使用全面的装饰器"""
from models import User, Transaction
# 验证操作数据
if not operation_data.get('user_id') or not operation_data.get('amount'):
raise ValueError("操作数据不完整")
user = session.query(User).filter(User.id == operation_data['user_id']).first()
if not user:
raise ValueError("用户不存在")
# 创建交易记录
transaction = Transaction(
user_id=user.id,
amount=operation_data['amount'],
type=operation_data.get('type', 'UNKNOWN'),
status='pending'
)
session.add(transaction)
session.flush()
# 执行业务逻辑
if operation_data['type'] == 'WITHDRAWAL':
if user.balance < operation_data['amount']:
raise ValueError("余额不足")
user.balance -= operation_data['amount']
elif operation_data['type'] == 'DEPOSIT':
user.balance += operation_data['amount']
transaction.status = 'completed'
session.add(user)
return {
'transaction_id': transaction.id,
'user_balance': user.balance
}
# 装饰器工厂函数
def transactional(session_factory: Callable = None,
isolation_level: str = "READ_COMMITTED",
timeout: Optional[float] = None,
max_retries: int = 0,
slow_threshold: float = 1.0,
enable_monitoring: bool = False,
propagation: PropagationBehavior = PropagationBehavior.REQUIRED):
"""灵活的事务装饰器工厂函数"""
def decorator(func: Callable) -> Callable:
decorators = []
# 基础事务装饰器
base_decorator = SQLAlchemyTransactionDecorator(
session_factory=session_factory,
isolation_level=isolation_level,
timeout=timeout
)
decorators.append(base_decorator)
# 重试装饰器
if max_retries > 0:
retry_decorator = RetryableTransactionDecorator(
session_factory=session_factory,
retry_config=RetryConfig(max_retries=max_retries)
)
decorators.append(retry_decorator)
# 监控装饰器
if enable_monitoring:
monitor_decorator = MonitoredTransactionDecorator(
session_factory=session_factory,
slow_threshold=slow_threshold
)
decorators.append(monitor_decorator)
# 嵌套事务装饰器
nested_decorator = NestedTransactionDecorator(
session_factory=session_factory,
propagation=propagation
)
decorators.append(nested_decorator)
# 应用装饰器链
for decorator_func in reversed(decorators):
func = decorator_func(func)
return func
return decorator
# 最简使用方式
@transactional(SessionFactory, max_retries=3, enable_monitoring=True)
def simple_transaction_example(session):
"""简单事务示例"""
# 业务逻辑
pass
# 性能测试装饰器
def performance_test_decorator(iterations: int = 100):
"""性能测试装饰器"""
def decorator(func: Callable) -> Callable:
@wraps(func)
def wrapper(*args, **kwargs):
start_time = time.time()
errors = 0
for i in range(iterations):
try:
func(*args, **kwargs)
except Exception as e:
errors += 1
logging.error(f"第 {i+1} 次执行失败: {e}")
end_time = time.time()
total_time = end_time - start_time
print(f"性能测试结果:")
print(f" 总执行次数: {iterations}")
print(f" 成功次数: {iterations - errors}")
print(f" 失败次数: {errors}")
print(f" 总耗时: {total_time:.3f}秒")
print(f" 平均耗时: {total_time/iterations:.3f}秒")
print(f" 成功率: {(iterations-errors)/iterations*100:.2f}%")
return wrapper
return decorator
# 使用性能测试
@performance_test_decorator(iterations=50)
@transactional(SessionFactory, max_retries=2)
def performance_test_operation(session):
"""性能测试操作"""
# 执行数据库操作
pass
5.5 嵌套事务处理
01.嵌套事务概述
a.基本概念
嵌套事务是指在现有事务内部创建子事务，实现更细粒度的事务控制。子事务可以独立提交或回滚，但其最终状态依赖于父事务的提交结果。
b.应用场景
适用于复杂业务逻辑处理，如订单处理包含多个步骤、批量操作中的部分失败处理、复杂的数据迁移过程等。嵌套事务能够提高代码的模块化和错误处理能力。
c.实现原理
基于数据库的保存点机制实现，每个子事务对应一个保存点。通过保存点的创建、回滚和释放操作实现嵌套事务的部分回滚功能。
02.保存点机制
a.保存点创建
使用SAVEPOINT命令创建命名保存点，作为子事务的起点。保存点名称需要在事务内唯一，可以使用递增数字或描述性名称。
b.保存点回滚
ROLLBACK TO SAVEPOINT命令回滚到指定保存点，撤销该保存点之后的所有操作。回滚不会影响保存点之前的操作状态。
c.保存点释放
RELEASE SAVEPOINT命令释放保存点，释放相关资源。释放后的保存点不能再被回滚，但可以创建同名的新的保存点。
d.基础实现
---
# 保存点基础操作示例
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker
from contextlib import contextmanager
import logging
class NestedTransactionManager:
"""嵌套事务管理器"""
def __init__(self, session):
self.session = session
self.savepoints = []
self.savepoint_counter = 0
def begin_nested(self, name: str = None) -> str:
"""开始嵌套事务"""
if name is None:
self.savepoint_counter += 1
name = f"sp_{self.savepoint_counter}"
# 创建保存点
self.session.execute(text(f"SAVEPOINT {name}"))
self.savepoints.append(name)
logging.info(f"创建保存点: {name}")
return name
def rollback_to_savepoint(self, name: str = None):
"""回滚到指定保存点"""
if name is None and self.savepoints:
name = self.savepoints.pop()
elif name in self.savepoints:
# 移除该保存点之后的所有保存点
index = self.savepoints.index(name)
self.savepoints = self.savepoints[:index + 1]
if name:
self.session.execute(text(f"ROLLBACK TO SAVEPOINT {name}"))
logging.info(f"回滚到保存点: {name}")
def release_savepoint(self, name: str = None):
"""释放保存点"""
if name is None and self.savepoints:
name = self.savepoints.pop()
elif name in self.savepoints:
self.savepoints.remove(name)
if name:
self.session.execute(text(f"RELEASE SAVEPOINT {name}"))
logging.info(f"释放保存点: {name}")
@contextmanager
def nested_transaction(self, name: str = None):
"""嵌套事务上下文管理器"""
savepoint_name = self.begin_nested(name)
try:
yield savepoint_name
self.release_savepoint(savepoint_name)
logging.info(f"嵌套事务提交: {savepoint_name}")
except Exception as e:
self.rollback_to_savepoint(savepoint_name)
logging.error(f"嵌套事务回滚: {savepoint_name} - {e}")
raise
# 使用示例
def nested_transaction_example():
"""嵌套事务使用示例"""
engine = create_engine('postgresql://user:pass@localhost/testdb')
Session = sessionmaker(bind=engine)
session = Session()
try:
session.begin()
nested_manager = NestedTransactionManager(session)
# 外层事务操作
print("开始外层事务")
# 执行一些基础操作...
# 第一个嵌套事务
with nested_manager.nested_transaction("user_update"):
print("执行用户更新操作")
# 更新用户信息
# session.execute(text("UPDATE users SET name = '新名称' WHERE id = 1"))
# 第二个嵌套事务
with nested_manager.nested_transaction("order_process"):
print("执行订单处理操作")
# 处理订单
# session.execute(text("INSERT INTO orders ..."))
# 第三个嵌套事务（更深层次）
with nested_manager.nested_transaction("inventory_update"):
print("执行库存更新操作")
# 更新库存
# session.execute(text("UPDATE products SET stock = stock - 1 WHERE id = 1"))
# 模拟失败情况
# raise Exception("库存更新失败")
session.commit()
print("外层事务提交成功")
except Exception as e:
session.rollback()
print(f"外层事务回滚: {e}")
finally:
session.close()
---
03.SQLAlchemy嵌套事务
a.begin_nested方法
SQLAlchemy提供session.begin_nested()方法创建嵌套事务，返回保存点事务对象。支持with语句进行自动管理。
b.savepoint管理
通过session.begin_nested()返回的对象可以调用commit()和rollback()方法，对应保存点的提交和回滚操作。
c.事务嵌套深度
SQLAlchemy支持多层事务嵌套，每层事务都可以独立回滚。但需要注意嵌套深度对性能的影响。
d.SQLAlchemy实现
---
# SQLAlchemy嵌套事务实现示例
from sqlalchemy import create_engine, text
from sqlalchemy.orm import sessionmaker, Session
from contextlib import contextmanager
import traceback
class SQLANestedTransactionManager:
"""SQLAlchemy嵌套事务管理器"""
def __init__(self, session: Session):
self.session = session
self.nested_transactions = []
@contextmanager
def nested_transaction(self, name: str = None):
"""SQLAlchemy嵌套事务上下文管理器"""
nested_tx = self.session.begin_nested()
transaction_info = {
'name': name or f"nested_{len(self.nested_transactions)}",
'savepoint': nested_tx,
'start_time': time.time()
}
self.nested_transactions.append(transaction_info)
try:
logging.info(f"开始嵌套事务: {transaction_info['name']}")
yield nested_tx
nested_tx.commit()
logging.info(f"嵌套事务提交: {transaction_info['name']}")
except Exception as e:
nested_tx.rollback()
logging.error(f"嵌套事务回滚: {transaction_info['name']} - {e}")
raise
finally:
self.nested_transactions.remove(transaction_info)
def get_nested_transaction_depth(self) -> int:
"""获取当前嵌套事务深度"""
return len(self.nested_transactions)
def rollback_all_nested(self):
"""回滚所有嵌套事务"""
for transaction_info in reversed(self.nested_transactions):
try:
transaction_info['savepoint'].rollback()
logging.info(f"强制回滚嵌套事务: {transaction_info['name']}")
except Exception as e:
logging.error(f"回滚嵌套事务失败: {transaction_info['name']} - {e}")
# 复杂业务逻辑示例
def complex_order_processing_example():
"""复杂订单处理，使用SQLAlchemy嵌套事务"""
engine = create_engine('postgresql://user:pass@localhost/testdb')
Session = sessionmaker(bind=engine)
session = Session()
try:
session.begin()
nested_manager = SQLANestedTransactionManager(session)
# 步骤1: 创建订单主记录
print("步骤1: 创建订单主记录")
session.execute(text("""
INSERT INTO orders (user_id, total_amount, status)
VALUES (1, 1000.0, 'processing')
RETURNING id
"""))
order_id = session.scalar(text("SELECT lastval()"))
# 步骤2: 处理订单项（嵌套事务1）
with nested_manager.nested_transaction("order_items"):
print("步骤2: 处理订单项")
session.execute(text("""
INSERT INTO order_items (order_id, product_id, quantity, price)
VALUES (:order_id, 1, 2, 500.0), (:order_id, 2, 1, 0.0)
"""), {"order_id": order_id})
# 步骤2.1: 更新库存（嵌套事务2）
with nested_manager.nested_transaction("inventory_update"):
print("步骤2.1: 更新库存")
session.execute(text("""
UPDATE products SET stock = stock - 2 WHERE id = 1
"""))
session.execute(text("""
UPDATE products SET stock = stock - 1 WHERE id = 2
"""))
# 检查库存是否充足
stock_check = session.execute(text("""
SELECT SUM(stock) FROM products WHERE id IN (1, 2)
""")).scalar()
if stock_check < 0:
raise ValueError("库存不足，无法完成订单")
# 步骤3: 更新用户积分（嵌套事务3）
with nested_manager.nested_transaction("user_points"):
print("步骤3: 更新用户积分")
points_to_add = int(1000 * 0.1) # 10%积分
session.execute(text("""
UPDATE users SET points = points + :points WHERE id = 1
"""), {"points": points_to_add})
# 步骤4: 记录日志（嵌套事务4）
with nested_manager.nested_transaction("activity_log"):
print("步骤4: 记录活动日志")
session.execute(text("""
INSERT INTO activity_logs (user_id, activity_type, description)
VALUES (1, 'order_created', '创建订单 #{}')
""").format(order_id))
# 更新订单状态为已完成
session.execute(text("""
UPDATE orders SET status = 'completed' WHERE id = :order_id
"""), {"order_id": order_id})
session.commit()
print(f"订单 {order_id} 处理完成")
except Exception as e:
session.rollback()
print(f"订单处理失败: {e}")
traceback.print_exc()
finally:
session.close()
---
04.Django嵌套事务
a.atomic装饰器嵌套
Django的@atomic装饰器支持嵌套使用，内层装饰器会创建保存点。可以通过savepoint参数控制是否使用保存点。
b.transaction.atomic嵌套
使用with transaction.atomic()语句进行嵌套事务控制。支持savepoint=False参数禁用保存点机制。
c.savepoint操作
Django提供transaction.savepoint()、transaction.savepoint_commit()、transaction.savepoint_rollback()等函数进行保存点操作。
d.Django实现
---
# Django嵌套事务实现示例
from django.db import transaction, models
from django.core.exceptions import ValidationError
import logging
class Order(models.Model):
status = models.CharField(max_length=20, default='pending')
total_amount = models.DecimalField(max_digits=10, decimal_places=2)
user = models.ForeignKey('auth.User', on_delete=models.CASCADE)
class OrderItem(models.Model):
order = models.ForeignKey(Order, on_delete=models.CASCADE)
product = models.ForeignKey('Product', on_delete=models.CASCADE)
quantity = models.PositiveIntegerField()
price = models.DecimalField(max_digits=10, decimal_places=2)
class Product(models.Model):
name = models.CharField(max_length=100)
stock = models.PositiveIntegerField(default=0)
price = models.DecimalField(max_digits=10, decimal_places=2)
class DjangoNestedTransactionManager:
"""Django嵌套事务管理器"""
@staticmethod
@transaction.atomic
def create_complex_order(user, items_data):
"""创建复杂订单，使用嵌套事务"""
try:
logging.info("开始创建复杂订单")
# 创建主订单记录
order = Order.objects.create(
user=user,
total_amount=0,
status='processing'
)
logging.info(f"创建订单: {order.id}")
# 处理订单项（嵌套事务1）
try:
with transaction.atomic():
total_amount = 0
for item_data in items_data:
# 检查库存并锁定产品
product = Product.objects.select_for_update().get(
id=item_data['product_id']
)
if product.stock < item_data['quantity']:
raise ValidationError(
f"产品 {product.name} 库存不足"
)
# 创建订单项
OrderItem.objects.create(
order=order,
product=product,
quantity=item_data['quantity'],
price=product.price
)
total_amount += product.price * item_data['quantity']
# 扣减库存（嵌套事务2）
DjangoNestedTransactionManager._update_inventory(items_data)
# 更新订单总金额
order.total_amount = total_amount
order.save()
# 记录用户活动（嵌套事务3）
DjangoNestedTransactionManager._log_user_activity(
user, f"创建订单 #{order.id}"
)
except ValidationError as e:
# 库存不足等业务异常
order.status = 'failed'
order.save()
raise e
except Exception as e:
# 其他异常
order.status = 'error'
order.save()
logging.error(f"处理订单项时发生错误: {e}")
raise e
# 更新订单状态为已完成
order.status = 'completed'
order.save()
logging.info(f"订单 {order.id} 创建成功")
return order
except Exception as e:
logging.error(f"创建订单失败: {e}")
raise
@staticmethod
@transaction.atomic(savepoint=False) # 创建新事务
def _update_inventory(items_data):
"""更新库存（总是新事务）"""
for item_data in items_data:
product = Product.objects.select_for_update().get(
id=item_data['product_id']
)
product.stock -= item_data['quantity']
product.save()
logging.info(f"更新库存: {product.name} 剩余 {product.stock}")
@staticmethod
@transaction.atomic # 可加入现有事务
def _log_user_activity(user, activity):
"""记录用户活动"""
from .models import UserActivityLog
UserActivityLog.objects.create(
user=user,
activity=activity
)
logging.info(f"记录用户活动: {user.username} - {activity}")
# 手动保存点操作示例
@staticmethod
def manual_savepoint_example():
"""手动保存点操作示例"""
try:
with transaction.atomic():
# 主事务操作
user = User.objects.get(id=1)
user.balance += 1000
user.save()
# 创建保存点
sp1 = transaction.savepoint("before_risky_operation")
logging.info("创建保存点: sp1")
try:
# 风险操作
risky_result = DjangoNestedTransactionManager._risky_operation()
# 如果成功，提交保存点
transaction.savepoint_commit(sp1)
logging.info("提交保存点: sp1")
except Exception as e:
# 如果失败，回滚到保存点
transaction.savepoint_rollback(sp1)
logging.warning(f"回滚到保存点: sp1 - {e}")
# 尝试备用操作
DjangoNestedTransactionManager._fallback_operation()
except Exception as e:
logging.error(f"主事务失败: {e}")
raise
@staticmethod
def _risky_operation():
"""风险操作"""
# 模拟可能失败的操作
import random
if random.random() < 0.3: # 30%概率失败
raise Exception("风险操作失败")
return "操作成功"
@staticmethod
def _fallback_operation():
"""备用操作"""
logging.info("执行备用操作")
# 使用示例
def django_nested_transaction_demo():
"""Django嵌套事务演示"""
from django.contrib.auth.models import User
try:
user = User.objects.get(id=1)
items_data = [
{'product_id': 1, 'quantity': 2},
{'product_id': 2, 'quantity': 1}
]
order = DjangoNestedTransactionManager.create_complex_order(user, items_data)
print(f"订单创建成功: {order.id}")
except ValidationError as e:
print(f"业务验证失败: {e}")
except Exception as e:
print(f"系统错误: {e}")
---
05.嵌套事务模式
a.补偿事务模式
在嵌套事务失败时，通过补偿操作撤销已完成的部分操作，实现数据一致性。补偿操作需要保证幂等性。
b.分支事务模式
将复杂业务拆分为多个独立的事务分支，每个分支可以独立提交或回滚。通过协调器确保所有分支的一致性。
c.模式实现
---
# 嵌套事务模式实现示例
from enum import Enum
from typing import List, Callable, Any
import time
class TransactionStatus(Enum):
PENDING = "pending"
COMMITTED = "committed"
ROLLED_BACK = "rolled_back"
COMPENSATED = "compensated"
class CompensationAction:
"""补偿动作"""
def __init__(self, name: str, action: Callable, *args, **kwargs):
self.name = name
self.action = action
self.args = args
self.kwargs = kwargs
self.executed = False
def execute(self):
"""执行补偿动作"""
if not self.executed:
try:
self.action(*self.args, **self.kwargs)
self.executed = True
return True
except Exception as e:
logging.error(f"补偿动作失败: {self.name} - {e}")
return False
return True
class CompensationTransactionManager:
"""补偿事务管理器"""
def __init__(self):
self.transactions = []
self.compensation_actions = []
def add_transaction(self, name: str, transaction_func: Callable,
compensation_func: Callable = None, *args, **kwargs):
"""添加事务和对应的补偿动作"""
transaction_info = {
'name': name,
'func': transaction_func,
'args': args,
'kwargs': kwargs,
'status': TransactionStatus.PENDING
}
self.transactions.append(transaction_info)
if compensation_func:
compensation = CompensationAction(
f"compensate_{name}",
compensation_func,
*args, **kwargs
)
self.compensation_actions.append(compensation)
def execute(self) -> bool:
"""执行所有事务"""
success_count = 0
for i, transaction in enumerate(self.transactions):
try:
logging.info(f"执行事务 {i+1}/{len(self.transactions)}: {transaction['name']}")
result = transaction['func'](*transaction['args'], **transaction['kwargs'])
transaction['status'] = TransactionStatus.COMMITTED
transaction['result'] = result
success_count += 1
logging.info(f"事务成功: {transaction['name']}")
except Exception as e:
transaction['status'] = TransactionStatus.ROLLED_BACK
logging.error(f"事务失败: {transaction['name']} - {e}")
# 执行补偿
self._execute_compensations(i)
return False
logging.info(f"所有事务执行成功，共 {success_count} 个")
return True
def _execute_compensations(self, failed_index: int):
"""执行补偿动作"""
logging.info(f"开始执行补偿，失败位置: {failed_index}")
# 按相反顺序执行补偿
for compensation in reversed(self.compensation_actions[:failed_index]):
if compensation.execute():
logging.info(f"补偿成功: {compensation.name}")
else:
logging.error(f"补偿失败: {compensation.name}")
def get_transaction_status(self) -> dict:
"""获取事务状态"""
return {
'transactions': [
{
'name': tx['name'],
'status': tx['status'].value,
'result': tx.get('result')
}
for tx in self.transactions
],
'compensation_actions': [
{
'name': ca.name,
'executed': ca.executed
}
for ca in self.compensation_actions
]
}
# 分支事务管理器
class BranchTransactionManager:
"""分支事务管理器"""
def __init__(self):
self.branches = []
def add_branch(self, name: str, transaction_func: Callable, *args, **kwargs):
"""添加事务分支"""
branch = {
'name': name,
'func': transaction_func,
'args': args,
'kwargs': kwargs,
'status': TransactionStatus.PENDING,
'result': None
}
self.branches.append(branch)
def execute_all_or_none(self) -> bool:
"""执行所有分支，全成功或全失败"""
executed_branches = []
try:
# 执行所有分支
for branch in self.branches:
logging.info(f"执行分支: {branch['name']}")
result = branch['func'](*branch['args'], **branch['kwargs'])
branch['result'] = result
branch['status'] = TransactionStatus.COMMITTED
executed_branches.append(branch)
logging.info("所有分支执行成功")
return True
except Exception as e:
logging.error(f"分支执行失败: {e}")
# 回滚已执行的分支
self._rollback_branches(executed_branches)
return False
def execute_with_partial_success(self) -> dict:
"""执行分支，允许部分成功"""
results = {
'successful': [],
'failed': []
}
for branch in self.branches:
try:
logging.info(f"执行分支: {branch['name']}")
result = branch['func'](*branch['args'], **branch['kwargs'])
branch['result'] = result
branch['status'] = TransactionStatus.COMMITTED
results['successful'].append(branch)
except Exception as e:
branch['status'] = TransactionStatus.ROLLED_BACK
branch['error'] = str(e)
results['failed'].append(branch)
logging.error(f"分支失败: {branch['name']} - {e}")
return results
def _rollback_branches(self, branches: List[dict]):
"""回滚指定分支"""
for branch in reversed(branches):
try:
# 这里假设每个分支函数都有对应的回滚方法
rollback_func = getattr(branch['func'], 'rollback', None)
if rollback_func:
rollback_func(*branch['args'], **branch['kwargs'])
branch['status'] = TransactionStatus.COMPENSATED
logging.info(f"分支回滚成功: {branch['name']}")
else:
logging.warning(f"分支 {branch['name']} 没有回滚方法")
except Exception as e:
logging.error(f"分支回滚失败: {branch['name']} - {e}")
# 使用示例
def compensation_transaction_example():
"""补偿事务使用示例"""
manager = CompensationTransactionManager()
# 定义事务和补偿动作
def create_order(user_id, amount):
print(f"创建订单: 用户{user_id}, 金额{amount}")
# 模拟订单创建
return {'order_id': 123, 'user_id': user_id, 'amount': amount}
def compensate_create_order(user_id, amount):
print(f"补偿: 删除订单 用户{user_id}, 金额{amount}")
def update_balance(user_id, amount):
print(f"更新余额: 用户{user_id}, 金额{-amount}")
# 模拟余额扣减
def compensate_update_balance(user_id, amount):
print(f"补偿: 恢复余额 用户{user_id}, 金额{amount}")
def send_notification(user_id, message):
print(f"发送通知: 用户{user_id}, 消息{message}")
# 添加事务
manager.add_transaction("create_order", create_order, compensate_create_order, 1, 1000)
manager.add_transaction("update_balance", update_balance, compensate_update_balance, 1, 1000)
manager.add_transaction("send_notification", send_notification, None, 1, "订单创建成功")
# 执行事务
success = manager.execute()
# 查看状态
status = manager.get_transaction_status()
print(f"事务执行结果: {success}")
print(f"事务状态: {status}")
def branch_transaction_example():
"""分支事务使用示例"""
manager = BranchTransactionManager()
def process_payment(order_id, amount):
print(f"处理支付: 订单{order_id}, 金额{amount}")
return {'payment_id': 'pay_123', 'status': 'success'}
def update_inventory(product_id, quantity):
print(f"更新库存: 产品{product_id}, 数量{-quantity}")
if quantity > 100: # 模拟库存不足
raise Exception("库存不足")
return {'remaining_stock': 50}
def schedule_shipping(order_id):
print(f"安排配送: 订单{order_id}")
return {'shipping_id': 'ship_456'}
# 添加分支
manager.add_branch("payment", process_payment, "order_123", 1000)
manager.add_branch("inventory", update_inventory, "prod_1", 50)
manager.add_branch("shipping", schedule_shipping, "order_123")
# 全部成功或全部失败模式
success = manager.execute_all_or_none()
print(f"分支事务结果: {success}")
# 部分成功模式
results = manager.execute_with_partial_success()
print(f"部分成功结果: {len(results['successful'])} 成功, {len(results['failed'])} 失败")
---
06.性能考虑与优化
a.嵌套深度控制
过深的嵌套会影响性能和可维护性，建议嵌套深度不超过3-4层。使用合理的代码结构减少不必要的嵌套。
b.资源管理
嵌套事务会占用更多数据库资源，包括锁、内存、连接等。需要及时释放不再需要的保存点。
c.监控与调试
实现嵌套事务的监控机制，记录每个嵌套层的执行时间和状态。提供详细的错误信息和调试日志。
d.优化实现
---
# 嵌套事务性能优化示例
import time
import threading
from typing import Dict, List
from dataclasses import dataclass
from collections import defaultdict
@dataclass
class NestedTransactionMetrics:
"""嵌套事务指标"""
name: str
start_time: float
end_time: float = None
duration: float = None
status: str = "pending"
children_count: int = 0
rollback_count: int = 0
class OptimizedNestedTransactionManager:
"""优化的嵌套事务管理器"""
def __init__(self, session, max_nested_depth: int = 3):
self.session = session
self.max_nested_depth = max_nested_depth
self.current_depth = 0
self.metrics_stack: List[NestedTransactionMetrics] = []
self.performance_stats = defaultdict(list)
self.lock = threading.Lock()
@contextmanager
def optimized_nested_transaction(self, name: str = None, timeout: float = 30.0):
"""优化的嵌套事务上下文管理器"""
if self.current_depth >= self.max_nested_depth:
raise RuntimeError(f"嵌套深度超过限制: {self.max_nested_depth}")
self.current_depth += 1
transaction_name = name or f"nested_tx_{self.current_depth}"
# 记录指标
metrics = NestedTransactionMetrics(
name=transaction_name,
start_time=time.time()
)
self.metrics_stack.append(metrics)
try:
# 检查超时
start_time = time.time()
with self.session.begin_nested():
yield transaction_name
# 检查是否超时
if time.time() - start_time > timeout:
raise TimeoutError(f"嵌套事务超时: {timeout}秒")
# 更新成功指标
metrics.end_time = time.time()
metrics.duration = metrics.end_time - metrics.start_time
metrics.status = "committed"
self.performance_stats['successful_transactions'].append(metrics)
except Exception as e:
# 更新失败指标
metrics.end_time = time.time()
metrics.duration = metrics.end_time - metrics.start_time
metrics.status = "rolled_back"
metrics.rollback_count += 1
self.performance_stats['failed_transactions'].append(metrics)
raise
finally:
self.current_depth -= 1
self.metrics_stack.pop()
def get_performance_report(self) -> Dict:
"""获取性能报告"""
with self.lock:
successful = self.performance_stats['successful_transactions']
failed = self.performance_stats['failed_transactions']
report = {
'total_transactions': len(successful) + len(failed),
'successful_count': len(successful),
'failed_count': len(failed),
'success_rate': len(successful) / (len(successful) + len(failed)) * 100 if (len(successful) + len(failed)) > 0 else 0,
'max_nested_depth': self.max_nested_depth,
'average_depth_used': self._calculate_average_depth()
}
if successful:
durations = [tx.duration for tx in successful]
report.update({
'average_duration': sum(durations) / len(durations),
'min_duration': min(durations),
'max_duration': max(durations)
})
if failed:
report['failure_reasons'] = self._analyze_failures(failed)
return report
def _calculate_average_depth(self) -> float:
"""计算平均使用的嵌套深度"""
if not self.performance_stats['successful_transactions']:
return 0
return sum(tx.children_count for tx in self.performance_stats['successful_transactions']) / len(self.performance_stats['successful_transactions'])
def _analyze_failures(self, failed_transactions: List[NestedTransactionMetrics]) -> Dict:
"""分析失败原因"""
failure_reasons = defaultdict(int)
for tx in failed_transactions:
# 这里可以根据实际情况分析失败原因
if tx.duration > 10:
failure_reasons['timeout'] += 1
elif tx.rollback_count > 1:
failure_reasons['multiple_rollbacks'] += 1
else:
failure_reasons['other'] += 1
return dict(failure_reasons)
def clear_stats(self):
"""清理统计信息"""
with self.lock:
self.performance_stats.clear()
# 嵌套事务调试工具
class NestedTransactionDebugger:
"""嵌套事务调试器"""
def __init__(self):
self.call_stack = []
self.log_entries = []
def log_transaction_start(self, name: str, depth: int):
"""记录事务开始"""
entry = {
'event': 'start',
'name': name,
'depth': depth,
'timestamp': time.time(),
'thread_id': threading.get_ident()
}
self.log_entries.append(entry)
self.call_stack.append(entry)
def log_transaction_end(self, name: str, depth: int, success: bool):
"""记录事务结束"""
entry = {
'event': 'end',
'name': name,
'depth': depth,
'success': success,
'timestamp': time.time(),
'thread_id': threading.get_ident()
}
self.log_entries.append(entry)
# 从调用栈中移除
self.call_stack = [e for e in self.call_stack if e['name'] != name]
def get_current_call_stack(self) -> List[str]:
"""获取当前调用栈"""
return [e['name'] for e in self.call_stack]
def print_debug_log(self):
"""打印调试日志"""
print("=== 嵌套事务调试日志 ===")
for entry in self.log_entries:
event_type = "开始" if entry['event'] == 'start' else "结束"
status = "成功" if entry.get('success', True) else "失败"
print(f"{event_type} 事务: {entry['name']} "
f"(深度: {entry['depth']}, "
f"线程: {entry['thread_id']}, "
f"时间: {time.strftime('%H:%M:%S', time.localtime(entry['timestamp']))})")
# 全局调试器实例
global_debugger = NestedTransactionDebugger()
# 带调试功能的嵌套事务管理器
class DebuggableNestedTransactionManager(OptimizedNestedTransactionManager):
"""可调试的嵌套事务管理器"""
@contextmanager
def debuggable_nested_transaction(self, name: str = None):
"""可调试的嵌套事务"""
transaction_name = name or f"debug_tx_{self.current_depth + 1}"
global_debugger.log_transaction_start(transaction_name, self.current_depth + 1)
try:
with self.optimized_nested_transaction(transaction_name):
yield transaction_name
global_debugger.log_transaction_end(transaction_name, self.current_depth + 1, True)
except Exception as e:
global_debugger.log_transaction_end(transaction_name, self.current_depth + 1, False)
raise
# 使用示例
def optimized_nested_transaction_demo():
"""优化的嵌套事务演示"""
engine = create_engine('postgresql://user:pass@localhost/testdb')
Session = sessionmaker(bind=engine)
session = Session()
try:
session.begin()
# 使用优化的嵌套事务管理器
manager = DebuggableNestedTransactionManager(session, max_nested_depth=3)
with manager.debuggable_nested_transaction("main_operation"):
with manager.debuggable_nested_transaction("step1"):
# 执行第一步操作
session.execute(text("SELECT 1"))
time.sleep(0.1) # 模拟操作
with manager.debuggable_nested_transaction("step2"):
# 执行第二步操作
session.execute(text("SELECT 2"))
time.sleep(0.2)
with manager.debuggable_nested_transaction("step2_1"):
# 更深的嵌套
session.execute(text("SELECT 3"))
time.sleep(0.05)
session.commit()
# 打印性能报告
report = manager.get_performance_report()
print("=== 性能报告 ===")
for key, value in report.items():
print(f"{key}: {value}")
# 打印调试日志
global_debugger.print_debug_log()
except Exception as e:
session.rollback()
print(f"操作失败: {e}")
global_debugger.print_debug_log()
finally:
session.close()
manager.clear_stats()
6. 分布式事务
6.1 两阶段提交2PC
01.两阶段提交概述
a.基本概念
两阶段提交是一种分布式事务协议，通过协调者协调多个参与者节点确保分布式事务的原子性。协议分为准备阶段和提交阶段，所有参与者要么全部提交，要么全部回滚，保证分布式系统的数据一致性。
b.协议特点
具有强一致性和原子性保证，但存在阻塞问题和单点故障风险。协调者负责全局决策，参与者负责本地资源管理，两者通过消息传递协调事务执行。
c.应用场景
适用于对一致性要求极高的分布式系统，如银行转账、库存管理等关键业务场景。在微服务架构中用于跨服务的数据操作一致性保证。
02.第一阶段：准备阶段
a.协调者请求
协调者向所有参与者发送准备请求，询问是否可以执行事务操作。参与者收到请求后开始执行本地事务，但不立即提交。
b.参与者响应
参与者检查本地资源是否可用，执行预提交操作，并锁定相关资源。如果可以提交则返回"就绪"响应，否则返回"中止"响应。
c.超时处理
参与者在规定时间内未收到协调者请求时，可以主动中止事务以释放资源。协调者等待所有参与者响应的超时机制确保系统可用性。
03.第二阶段：提交阶段
a.提交决策
协调者根据参与者的响应做出全局决策。如果所有参与者都返回"就绪"，协调者决定提交事务；如果有任何参与者返回"中止"或超时，则决定中止事务。
b.执行提交
协调者向所有参与者发送提交或中止请求。参与者收到提交请求后正式提交本地事务并释放资源，收到中止请求则回滚本地事务。
c.完成确认
参与者在执行完提交或中止操作后向协调者发送确认消息。协调者收到所有参与者的确认后，整个分布式事务完成。
04.故障处理
a.协调者故障
协调者在执行过程中出现故障时，参与者无法获得最终决策。参与者需要通过超时机制或与其他参与者通信来决定事务的最终状态。
b.参与者故障
参与者故障时协调者需要能够检测到故障并做出相应的处理。其他正常运行的参与者需要能够继续完成事务或进行回滚操作。
c.网络故障
网络分区或消息丢失可能导致协调者和参与者之间的通信中断。需要设计重试机制和状态恢复策略来处理网络故障。
05.实现示例
a.基础两阶段提交
---
# 基础两阶段提交实现
import logging
import time
from typing import Dict, List, Optional, Tuple
from enum import Enum
from dataclasses import dataclass
from threading import Thread, Event
from queue import Queue, Empty
class TransactionState(Enum):
"""事务状态枚举"""
INIT = "INIT"
PREPARING = "PREPARING"
PREPARED = "PREPARED"
COMMITTING = "COMMITTING"
COMMITTED = "COMMITTED"
ABORTING = "ABORTING"
ABORTED = "ABORTED"
class ParticipantResponse(Enum):
"""参与者响应枚举"""
READY = "READY"
ABORT = "ABORT"
TIMEOUT = "TIMEOUT"
@dataclass
class Transaction:
"""事务数据结构"""
transaction_id: str
participants: List[str]
operation_data: Dict
state: TransactionState = TransactionState.INIT
created_at: float = 0
class Participant:
"""参与者节点"""
def __init__(self, name: str):
self.name = name
self.local_transactions = {}
self.logger = logging.getLogger(f"Participant-{name}")
def prepare(self, transaction_id: str, operation_data: Dict) -> ParticipantResponse:
"""准备阶段"""
self.logger.info(f"准备事务 {transaction_id}")
try:
# 模拟本地资源检查和预提交
if self._check_resources(operation_data):
self.local_transactions[transaction_id] = {
'state': 'PREPARED',
'data': operation_data
}
self.logger.info(f"事务 {transaction_id} 准备成功")
return ParticipantResponse.READY
else:
self.logger.info(f"事务 {transaction_id} 资源不足，准备失败")
return ParticipantResponse.ABORT
except Exception as e:
self.logger.error(f"事务 {transaction_id} 准备异常: {e}")
return ParticipantResponse.ABORT
def commit(self, transaction_id: str) -> bool:
"""提交事务"""
self.logger.info(f"提交事务 {transaction_id}")
try:
if transaction_id in self.local_transactions:
# 模拟本地提交操作
self._execute_commit(transaction_id)
self.local_transactions[transaction_id]['state'] = 'COMMITTED'
self.logger.info(f"事务 {transaction_id} 提交成功")
return True
return False
except Exception as e:
self.logger.error(f"事务 {transaction_id} 提交异常: {e}")
return False
def abort(self, transaction_id: str) -> bool:
"""回滚事务"""
self.logger.info(f"回滚事务 {transaction_id}")
try:
if transaction_id in self.local_transactions:
# 模拟本地回滚操作
self._execute_abort(transaction_id)
del self.local_transactions[transaction_id]
self.logger.info(f"事务 {transaction_id} 回滚成功")
return True
return False
except Exception as e:
self.logger.error(f"事务 {transaction_id} 回滚异常: {e}")
return False
def _check_resources(self, operation_data: Dict) -> bool:
"""检查本地资源"""
# 模拟资源检查逻辑
return True
def _execute_commit(self, transaction_id: str):
"""执行本地提交"""
# 模拟提交操作
time.sleep(0.1)
def _execute_abort(self, transaction_id: str):
"""执行本地回滚"""
# 模拟回滚操作
time.sleep(0.1)
class Coordinator:
"""协调者节点"""
def __init__(self):
self.transactions = {}
self.participants = {}
self.logger = logging.getLogger("Coordinator")
def add_participant(self, participant: Participant):
"""添加参与者"""
self.participants[participant.name] = participant
def begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""开始两阶段提交事务"""
self.logger.info(f"开始两阶段提交事务 {transaction_id}")
transaction = Transaction(
transaction_id=transaction_id,
participants=participants,
operation_data=operation_data,
created_at=time.time()
)
self.transactions[transaction_id] = transaction
try:
# 第一阶段：准备阶段
if not self._prepare_phase(transaction):
# 准备失败，执行回滚
self._abort_phase(transaction)
return False
# 第二阶段：提交阶段
return self._commit_phase(transaction)
except Exception as e:
self.logger.error(f"事务 {transaction_id} 执行异常: {e}")
self._abort_phase(transaction)
return False
def _prepare_phase(self, transaction: Transaction) -> bool:
"""准备阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入准备阶段")
transaction.state = TransactionState.PREPARING
ready_count = 0
timeout = 10 # 10秒超时
# 向所有参与者发送准备请求
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
self.logger.error(f"参与者 {participant_name} 不存在")
continue
try:
response = participant.prepare(transaction.transaction_id, transaction.operation_data)
if response == ParticipantResponse.READY:
ready_count += 1
self.logger.info(f"参与者 {participant_name} 准备就绪")
else:
self.logger.warning(f"参与者 {participant_name} 准备失败: {response}")
return False
except Exception as e:
self.logger.error(f"参与者 {participant_name} 准备异常: {e}")
return False
# 检查是否所有参与者都准备就绪
if ready_count == len(transaction.participants):
transaction.state = TransactionState.PREPARED
self.logger.info(f"事务 {transaction.transaction_id} 准备阶段完成")
return True
else:
self.logger.warning(f"事务 {transaction.transaction_id} 准备阶段失败，准备就绪: {ready_count}/{len(transaction.participants)}")
return False
def _commit_phase(self, transaction: Transaction) -> bool:
"""提交阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入提交阶段")
transaction.state = TransactionState.COMMITTING
success_count = 0
# 向所有参与者发送提交请求
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
try:
if participant.commit(transaction.transaction_id):
success_count += 1
self.logger.info(f"参与者 {participant_name} 提交成功")
else:
self.logger.error(f"参与者 {participant_name} 提交失败")
# 注意：在2PC中，如果有参与者提交失败，系统可能处于不一致状态
except Exception as e:
self.logger.error(f"参与者 {participant_name} 提交异常: {e}")
if success_count == len(transaction.participants):
transaction.state = TransactionState.COMMITTED
self.logger.info(f"事务 {transaction.transaction_id} 提交完成")
return True
else:
transaction.state = TransactionState.ABORTED
self.logger.error(f"事务 {transaction.transaction_id} 提交失败，成功: {success_count}/{len(transaction.participants)}")
return False
def _abort_phase(self, transaction: Transaction):
"""中止阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入中止阶段")
transaction.state = TransactionState.ABORTING
# 向所有参与者发送中止请求
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
try:
participant.abort(transaction.transaction_id)
self.logger.info(f"参与者 {participant_name} 回滚成功")
except Exception as e:
self.logger.error(f"参与者 {participant_name} 回滚异常: {e}")
transaction.state = TransactionState.ABORTED
self.logger.info(f"事务 {transaction.transaction_id} 中止完成")
# 使用示例
def basic_2pc_example():
"""基础两阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建协调者
coordinator = Coordinator()
# 创建参与者
participant1 = Participant("BankServer1")
participant2 = Participant("BankServer2")
participant3 = Participant("InventoryServer")
# 注册参与者
coordinator.add_participant(participant1)
coordinator.add_participant(participant2)
coordinator.add_participant(participant3)
# 执行分布式转账事务
transaction_id = "TXN_001"
participants = ["BankServer1", "BankServer2", "InventoryServer"]
operation_data = {
"transfer_amount": 1000.0,
"from_account": "user001",
"to_account": "user002",
"product_update": "product123"
}
# 执行事务
success = coordinator.begin_transaction(transaction_id, participants, operation_data)
if success:
print(f"事务 {transaction_id} 执行成功")
else:
print(f"事务 {transaction_id} 执行失败")
---
b.超时和重试机制
---
# 带超时和重试机制的两阶段提交
import random
from threading import Timer
from concurrent.futures import ThreadPoolExecutor, as_completed
class RobustCoordinator(Coordinator):
"""带超时和重试机制的协调者"""
def __init__(self, max_retries=3, timeout=5):
super().__init__()
self.max_retries = max_retries
self.timeout = timeout
self.response_timeout = 3
def _send_request_with_timeout(self, participant, method_name, transaction_id, operation_data=None):
"""发送带超时的请求"""
try:
if method_name == "prepare":
response = participant.prepare(transaction_id, operation_data or {})
elif method_name == "commit":
response = participant.commit(transaction_id)
elif method_name == "abort":
response = participant.abort(transaction_id)
else:
raise ValueError(f"未知方法: {method_name}")
return response, None
except Exception as e:
return None, str(e)
def _prepare_with_retry(self, transaction: Transaction) -> bool:
"""带重试的准备阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 开始准备阶段（带重试）")
for attempt in range(self.max_retries):
self.logger.info(f"准备阶段尝试 {attempt + 1}/{self.max_retries}")
ready_responses = []
failed_participants = []
# 并行发送准备请求
with ThreadPoolExecutor(max_workers=len(transaction.participants)) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
failed_participants.append(participant_name)
continue
future = executor.submit(
self._send_request_with_timeout,
participant, "prepare", transaction.transaction_id, transaction.operation_data
)
future_to_participant[future] = participant_name
# 收集响应
for future in as_completed(future_to_participant, timeout=self.response_timeout):
participant_name = future_to_participant[future]
try:
response, error = future.result()
if error:
self.logger.error(f"参与者 {participant_name} 准备异常: {error}")
failed_participants.append(participant_name)
elif response == ParticipantResponse.READY:
ready_responses.append(participant_name)
self.logger.info(f"参与者 {participant_name} 准备就绪")
else:
self.logger.warning(f"参与者 {participant_name} 准备失败: {response}")
failed_participants.append(participant_name)
except Exception as e:
self.logger.error(f"参与者 {participant_name} 准备超时: {e}")
failed_participants.append(participant_name)
# 检查结果
if len(failed_participants) == 0 and len(ready_responses) == len(transaction.participants):
self.logger.info(f"事务 {transaction.transaction_id} 准备阶段成功（尝试 {attempt + 1}）")
return True
else:
self.logger.warning(f"事务 {transaction.transaction_id} 准备阶段失败（尝试 {attempt + 1}），失败参与者: {failed_participants}")
# 如果不是最后一次尝试，回滚已准备的参与者
if attempt < self.max_retries - 1:
self._abort_prepared_participants(ready_responses, transaction.transaction_id)
time.sleep(2 ** attempt) # 指数退避
return False
def _commit_with_retry(self, transaction: Transaction) -> bool:
"""带重试的提交阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 开始提交阶段（带重试）")
success_count = 0
failed_participants = []
# 并行发送提交请求
with ThreadPoolExecutor(max_workers=len(transaction.participants)) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
future = executor.submit(
self._send_request_with_timeout,
participant, "commit", transaction.transaction_id
)
future_to_participant[future] = participant_name
# 收集响应
for future in as_completed(future_to_participant, timeout=self.response_timeout):
participant_name = future_to_participant[future]
try:
success, error = future.result()
if error:
self.logger.error(f"参与者 {participant_name} 提交异常: {error}")
failed_participants.append(participant_name)
elif success:
success_count += 1
self.logger.info(f"参与者 {participant_name} 提交成功")
else:
self.logger.error(f"参与者 {participant_name} 提交失败")
failed_participants.append(participant_name)
except Exception as e:
self.logger.error(f"参与者 {participant_name} 提交超时: {e}")
failed_participants.append(participant_name)
# 检查结果
if success_count == len(transaction.participants):
self.logger.info(f"事务 {transaction.transaction_id} 提交阶段成功")
return True
else:
self.logger.error(f"事务 {transaction.transaction_id} 提交阶段失败，成功: {success_count}/{len(transaction.participants)}")
return False
def _abort_prepared_participants(self, prepared_participants: List[str], transaction_id: str):
"""中止已准备的参与者"""
for participant_name in prepared_participants:
participant = self.participants.get(participant_name)
if participant:
try:
participant.abort(transaction_id)
self.logger.info(f"回滚参与者 {participant_name} 的准备状态")
except Exception as e:
self.logger.error(f"回滚参与者 {participant_name} 失败: {e}")
def begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""重写事务开始方法"""
transaction = Transaction(
transaction_id=transaction_id,
participants=participants,
operation_data=operation_data,
created_at=time.time()
)
self.transactions[transaction_id] = transaction
try:
# 第一阶段：准备阶段（带重试）
if not self._prepare_with_retry(transaction):
self._abort_phase(transaction)
return False
# 第二阶段：提交阶段（带重试）
return self._commit_with_retry(transaction)
except Exception as e:
self.logger.error(f"事务 {transaction_id} 执行异常: {e}")
self._abort_phase(transaction)
return False
# 使用示例
def robust_2pc_example():
"""带重试机制的两阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建鲁棒的协调者
coordinator = RobustCoordinator(max_retries=3, timeout=5)
# 创建参与者
participant1 = Participant("OrderServer")
participant2 = Participant("PaymentServer")
participant3 = Participant("ShippingServer")
# 注册参与者
coordinator.add_participant(participant1)
coordinator.add_participant(participant2)
coordinator.add_participant(participant3)
# 执行订单处理事务
transaction_id = "ORDER_001"
participants = ["OrderServer", "PaymentServer", "ShippingServer"]
operation_data = {
"order_id": "ORD123",
"user_id": "USER456",
"amount": 299.99,
"items": ["item1", "item2"]
}
# 执行事务
success = coordinator.begin_transaction(transaction_id, participants, operation_data)
if success:
print(f"订单事务 {transaction_id} 执行成功")
else:
print(f"订单事务 {transaction_id} 执行失败")
---
c.状态持久化
---
# 带状态持久化的两阶段提交
import json
import os
from pathlib import Path
class TransactionStatePersistence:
"""事务状态持久化管理"""
def __init__(self, storage_path: str = "./transaction_states"):
self.storage_path = Path(storage_path)
self.storage_path.mkdir(exist_ok=True)
self.logger = logging.getLogger("StatePersistence")
def save_transaction_state(self, transaction: Transaction):
"""保存事务状态"""
state_file = self.storage_path / f"{transaction.transaction_id}.json"
try:
state_data = {
'transaction_id': transaction.transaction_id,
'participants': transaction.participants,
'operation_data': transaction.operation_data,
'state': transaction.state.value,
'created_at': transaction.created_at
}
with open(state_file, 'w', encoding='utf-8') as f:
json.dump(state_data, f, indent=2, ensure_ascii=False)
self.logger.info(f"事务状态已保存: {transaction.transaction_id}")
except Exception as e:
self.logger.error(f"保存事务状态失败: {e}")
def load_transaction_state(self, transaction_id: str) -> Optional[Transaction]:
"""加载事务状态"""
state_file = self.storage_path / f"{transaction_id}.json"
if not state_file.exists():
return None
try:
with open(state_file, 'r', encoding='utf-8') as f:
state_data = json.load(f)
transaction = Transaction(
transaction_id=state_data['transaction_id'],
participants=state_data['participants'],
operation_data=state_data['operation_data']
)
transaction.state = TransactionState(state_data['state'])
transaction.created_at = state_data['created_at']
self.logger.info(f"事务状态已加载: {transaction_id}")
return transaction
except Exception as e:
self.logger.error(f"加载事务状态失败: {e}")
return None
def delete_transaction_state(self, transaction_id: str):
"""删除事务状态文件"""
state_file = self.storage_path / f"{transaction_id}.json"
try:
if state_file.exists():
state_file.unlink()
self.logger.info(f"事务状态文件已删除: {transaction_id}")
except Exception as e:
self.logger.error(f"删除事务状态文件失败: {e}")
def get_all_pending_transactions(self) -> List[str]:
"""获取所有待处理的事务ID"""
pending_transactions = []
try:
for state_file in self.storage_path.glob("*.json"):
transaction_id = state_file.stem
transaction = self.load_transaction_state(transaction_id)
if transaction and transaction.state in [
TransactionState.PREPARING,
TransactionState.PREPARED,
TransactionState.COMMITTING
]:
pending_transactions.append(transaction_id)
except Exception as e:
self.logger.error(f"获取待处理事务失败: {e}")
return pending_transactions
class PersistentCoordinator(RobustCoordinator):
"""带状态持久化的协调者"""
def __init__(self, max_retries=3, timeout=5, storage_path="./transaction_states"):
super().__init__(max_retries, timeout)
self.persistence = TransactionStatePersistence(storage_path)
self.logger = logging.getLogger("PersistentCoordinator")
def begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""开始持久化事务"""
self.logger.info(f"开始持久化事务 {transaction_id}")
# 检查是否有未完成的相同事务
existing_transaction = self.persistence.load_transaction_state(transaction_id)
if existing_transaction:
self.logger.warning(f"发现未完成的事务 {transaction_id}，状态: {existing_transaction.state}")
# 根据状态决定是恢复还是回滚
if existing_transaction.state == TransactionState.PREPARED:
# 从准备状态继续执行
return self._continue_transaction(existing_transaction)
else:
# 回滚未完成的事务
self._abort_phase(existing_transaction)
self.persistence.delete_transaction_state(transaction_id)
# 创建新事务
transaction = Transaction(
transaction_id=transaction_id,
participants=participants,
operation_data=operation_data,
created_at=time.time()
)
self.transactions[transaction_id] = transaction
self.persistence.save_transaction_state(transaction)
try:
# 第一阶段：准备阶段
if not self._prepare_with_retry(transaction):
self._abort_phase(transaction)
self.persistence.delete_transaction_state(transaction_id)
return False
# 更新状态
self.persistence.save_transaction_state(transaction)
# 第二阶段：提交阶段
success = self._commit_with_retry(transaction)
if success:
# 事务成功，删除状态文件
self.persistence.delete_transaction_state(transaction_id)
else:
# 更新失败状态
self.persistence.save_transaction_state(transaction)
return success
except Exception as e:
self.logger.error(f"持久化事务 {transaction_id} 执行异常: {e}")
self._abort_phase(transaction)
self.persistence.delete_transaction_state(transaction_id)
return False
def _continue_transaction(self, transaction: Transaction) -> bool:
"""继续未完成的事务"""
self.logger.info(f"继续执行未完成的事务 {transaction.transaction_id}")
try:
# 从准备状态继续执行提交阶段
success = self._commit_with_retry(transaction)
if success:
self.persistence.delete_transaction_state(transaction.transaction_id)
self.logger.info(f"事务 {transaction.transaction_id} 恢复成功")
else:
self.persistence.save_transaction_state(transaction)
self.logger.error(f"事务 {transaction.transaction_id} 恢复失败")
return success
except Exception as e:
self.logger.error(f"事务 {transaction.transaction_id} 恢复异常: {e}")
self._abort_phase(transaction)
self.persistence.delete_transaction_state(transaction.transaction_id)
return False
def recover_transactions(self):
"""恢复未完成的事务"""
self.logger.info("开始恢复未完成的事务")
pending_transactions = self.persistence.get_all_pending_transactions()
for transaction_id in pending_transactions:
self.logger.info(f"恢复事务: {transaction_id}")
transaction = self.persistence.load_transaction_state(transaction_id)
if transaction:
if transaction.state == TransactionState.PREPARED:
self._continue_transaction(transaction)
else:
# 状态异常，直接回滚
self._abort_phase(transaction)
self.persistence.delete_transaction_state(transaction_id)
# 使用示例
def persistent_2pc_example():
"""持久化两阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建持久化协调者
coordinator = PersistentCoordinator(
max_retries=3,
timeout=5,
storage_path="./transaction_states"
)
# 创建参与者
participants = [
Participant("UserService"),
Participant("OrderService"),
Participant("InventoryService")
]
for participant in participants:
coordinator.add_participant(participant)
# 执行用户注册事务
transaction_id = "USER_REG_001"
participants_list = ["UserService", "OrderService", "InventoryService"]
operation_data = {
"user_id": "newuser123",
"username": "john_doe",
"email": "[email protected]",
"initial_order": True
}
# 模拟协调者在执行过程中重启
try:
success = coordinator.begin_transaction(transaction_id, participants_list, operation_data)
if success:
print(f"用户注册事务 {transaction_id} 执行成功")
else:
print(f"用户注册事务 {transaction_id} 执行失败")
except KeyboardInterrupt:
print("模拟协调者崩溃...")
# 模拟协调者重启后恢复
print("协调者重启，恢复未完成的事务...")
coordinator.recover_transactions()
---
06.性能优化
a.并行处理
使用多线程或异步IO并行处理参与者的请求，减少整体事务执行时间。合理设置线程池大小，避免资源竞争和上下文切换开销。
b.批量事务
将多个小事务合并为一个大事务，减少协调者和参与者之间的通信次数。需要合理设计批量大小和事务边界，避免长时间锁定资源。
c.缓存机制
使用缓存存储常用的事务状态和参与者信息，减少磁盘IO和网络通信。实现本地缓存和分布式缓存的一致性策略。
6.2 三阶段提交3PC
01.三阶段提交概述
a.基本概念
三阶段提交是对两阶段提交协议的改进，通过增加预提交阶段解决阻塞问题。协议分为CanCommit、PreCommit和DoCommit三个阶段，提供更好的容错性和非阻塞特性。
b.协议改进
相比两阶段提交，三阶段提交消除了协调者单点故障导致的阻塞问题。参与者可以在协调者故障时自主做出决策，避免了无限等待的情况。
c.适用场景
适用于对可用性要求较高的分布式系统，特别是在网络分区和节点故障较为常见的环境中。相比两阶段提交牺牲了一定的性能来换取更好的容错性。
02.第一阶段：CanCommit
a.协调者询问
协调者向所有参与者发送CanCommit请求，询问事务是否可以执行。参与者收到请求后进行资源检查和可行性分析，但不会实际执行事务操作。
b.参与者响应
参与者检查本地资源是否可用，验证事务执行条件。如果可以执行则返回Yes响应，否则返回No响应。此阶段不会锁定资源。
c.快速决策
如果任何参与者返回No，协调者立即决定中止事务并通知所有参与者执行Abort操作。如果所有参与者都返回Yes，则进入PreCommit阶段。
03.第二阶段：PreCommit
a.预提交准备
协调者向所有参与者发送PreCommit请求，参与者开始执行事务的预处理操作但不最终提交。参与者锁定所需资源并执行操作。
b.超时机制
参与者在PreCommit阶段启动超时计时器，如果在规定时间内未收到协调者的最终决策，会自动进入Abort状态释放资源。
c.故障容错
如果协调者在PreCommit阶段发生故障，参与者之间可以通过通信协议自主决策，避免无限等待。这是三阶段提交的关键改进。
04.第三阶段：DoCommit
a.最终决策
协调者根据PreCommit阶段的响应做出最终提交决策。如果所有参与者都成功预处理，则发送Commit请求；否则发送Abort请求。
b.执行提交
参与者收到Commit请求后正式提交事务，释放资源并向协调者发送确认。收到Abort请求则回滚事务并释放资源。
c.完成确认
协调者等待所有参与者的确认响应，确认事务完成。如果有参与者未响应，协调者会重试或使用其他机制确保一致性。
05.实现示例
a.基础三阶段提交
---
# 基础三阶段提交实现
import logging
import time
import threading
from typing import Dict, List, Optional, Set
from enum import Enum
from dataclasses import dataclass, field
from concurrent.futures import ThreadPoolExecutor, as_completed
class ThreePCState(Enum):
"""三阶段提交状态枚举"""
INIT = "INIT"
CAN_COMMIT = "CAN_COMMIT"
PRE_COMMIT = "PRE_COMMIT"
DO_COMMIT = "DO_COMMIT"
COMMITTED = "COMMITTED"
ABORTED = "ABORTED"
TIMEOUT_ABORT = "TIMEOUT_ABORT"
class ThreePCResponse(Enum):
"""三阶段提交响应枚举"""
YES = "YES"
NO = "NO"
ACK = "ACK"
TIMEOUT = "TIMEOUT"
@dataclass
class ThreePCTransaction:
"""三阶段提交事务数据结构"""
transaction_id: str
participants: List[str]
operation_data: Dict
state: ThreePCState = ThreePCState.INIT
created_at: float = field(default_factory=time.time)
timeout: int = 10
votes: Dict[str, ThreePCResponse] = field(default_factory=dict)
acknowledgements: Dict[str, ThreePCResponse] = field(default_factory=dict)
class ThreePCParticipant:
"""三阶段提交参与者"""
def __init__(self, name: str, timeout: int = 5):
self.name = name
self.timeout = timeout
self.local_transactions = {}
self.locked_resources = {}
self.logger = logging.getLogger(f"ThreePCParticipant-{name}")
self.timers = {}
def can_commit(self, transaction_id: str, operation_data: Dict) -> ThreePCResponse:
"""CanCommit阶段"""
self.logger.info(f"CanCommit阶段: {transaction_id}")
try:
# 检查资源可用性
if self._can_commit_resources(operation_data):
self.local_transactions[transaction_id] = {
'state': 'CAN_COMMIT',
'data': operation_data,
'can_commit_time': time.time()
}
self.logger.info(f"事务 {transaction_id} CanCommit: YES")
return ThreePCResponse.YES
else:
self.logger.info(f"事务 {transaction_id} CanCommit: NO")
return ThreePCResponse.NO
except Exception as e:
self.logger.error(f"事务 {transaction_id} CanCommit异常: {e}")
return ThreePCResponse.NO
def pre_commit(self, transaction_id: str) -> ThreePCResponse:
"""PreCommit阶段"""
self.logger.info(f"PreCommit阶段: {transaction_id}")
try:
if transaction_id in self.local_transactions:
# 锁定资源并执行预处理
if self._lock_resources(transaction_id):
if self._execute_pre_commit(transaction_id):
self.local_transactions[transaction_id]['state'] = 'PRE_COMMIT'
self.local_transactions[transaction_id]['pre_commit_time'] = time.time()
# 启动超时定时器
self._start_timeout_timer(transaction_id)
self.logger.info(f"事务 {transaction_id} PreCommit: ACK")
return ThreePCResponse.ACK
else:
self._release_resources(transaction_id)
del self.local_transactions[transaction_id]
return ThreePCResponse.NO
else:
return ThreePCResponse.NO
else:
self.logger.error(f"事务 {transaction_id} 未在CanCommit阶段注册")
return ThreePCResponse.NO
except Exception as e:
self.logger.error(f"事务 {transaction_id} PreCommit异常: {e}")
self._release_resources(transaction_id)
return ThreePCResponse.NO
def do_commit(self, transaction_id: str) -> ThreePCResponse:
"""DoCommit阶段"""
self.logger.info(f"DoCommit阶段: {transaction_id}")
try:
if transaction_id in self.local_transactions:
# 取消超时定时器
self._cancel_timeout_timer(transaction_id)
# 执行最终提交
if self._execute_commit(transaction_id):
self.local_transactions[transaction_id]['state'] = 'COMMITTED'
self.local_transactions[transaction_id]['commit_time'] = time.time()
self._release_resources(transaction_id)
del self.local_transactions[transaction_id]
self.logger.info(f"事务 {transaction_id} DoCommit: ACK")
return ThreePCResponse.ACK
else:
# 提交失败，回滚
self._rollback_transaction(transaction_id)
return ThreePCResponse.NO
else:
self.logger.error(f"事务 {transaction_id} 未在PreCommit阶段注册")
return ThreePCResponse.NO
except Exception as e:
self.logger.error(f"事务 {transaction_id} DoCommit异常: {e}")
self._rollback_transaction(transaction_id)
return ThreePCResponse.NO
def abort_transaction(self, transaction_id: str):
"""中止事务"""
self.logger.info(f"中止事务: {transaction_id}")
try:
self._cancel_timeout_timer(transaction_id)
self._rollback_transaction(transaction_id)
except Exception as e:
self.logger.error(f"中止事务 {transaction_id} 异常: {e}")
def _can_commit_resources(self, operation_data: Dict) -> bool:
"""检查资源是否可用于提交"""
# 模拟资源检查逻辑
return True
def _lock_resources(self, transaction_id: str) -> bool:
"""锁定事务所需资源"""
if transaction_id in self.local_transactions:
self.locked_resources[transaction_id] = {
'locked_at': time.time(),
'resources': ['resource1', 'resource2'] # 模拟资源
}
return True
return False
def _release_resources(self, transaction_id: str):
"""释放事务资源"""
if transaction_id in self.locked_resources:
del self.locked_resources[transaction_id]
def _execute_pre_commit(self, transaction_id: str) -> bool:
"""执行预处理操作"""
# 模拟预处理操作
time.sleep(0.1)
return True
def _execute_commit(self, transaction_id: str) -> bool:
"""执行最终提交"""
# 模拟提交操作
time.sleep(0.1)
return True
def _rollback_transaction(self, transaction_id: str):
"""回滚事务"""
if transaction_id in self.local_transactions:
del self.local_transactions[transaction_id]
self._release_resources(transaction_id)
def _start_timeout_timer(self, transaction_id: str):
"""启动超时定时器"""
timer = threading.Timer(
self.timeout,
self._handle_timeout,
args=[transaction_id]
)
self.timers[transaction_id] = timer
timer.start()
def _cancel_timeout_timer(self, transaction_id: str):
"""取消超时定时器"""
if transaction_id in self.timers:
self.timers[transaction_id].cancel()
del self.timers[transaction_id]
def _handle_timeout(self, transaction_id: str):
"""处理超时情况"""
self.logger.warning(f"事务 {transaction_id} 超时，自动中止")
self._rollback_transaction(transaction_id)
class ThreePCCoordinator:
"""三阶段提交协调者"""
def __init__(self, timeout: int = 10, max_retries: int = 3):
self.transactions = {}
self.participants = {}
self.timeout = timeout
self.max_retries = max_retries
self.logger = logging.getLogger("ThreePCCoordinator")
def add_participant(self, participant: ThreePCParticipant):
"""添加参与者"""
self.participants[participant.name] = participant
def begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""开始三阶段提交事务"""
self.logger.info(f"开始三阶段提交事务: {transaction_id}")
transaction = ThreePCTransaction(
transaction_id=transaction_id,
participants=participants,
operation_data=operation_data,
timeout=self.timeout
)
self.transactions[transaction_id] = transaction
try:
# 第一阶段：CanCommit
if not self._can_commit_phase(transaction):
self._abort_transaction(transaction)
return False
# 第二阶段：PreCommit
if not self._pre_commit_phase(transaction):
self._abort_transaction(transaction)
return False
# 第三阶段：DoCommit
return self._do_commit_phase(transaction)
except Exception as e:
self.logger.error(f"事务 {transaction_id} 执行异常: {e}")
self._abort_transaction(transaction)
return False
def _can_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""CanCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入CanCommit阶段")
transaction.state = ThreePCState.CAN_COMMIT
yes_votes = 0
no_votes = 0
# 并行发送CanCommit请求
with ThreadPoolExecutor(max_workers=len(transaction.participants)) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
future = executor.submit(
participant.can_commit,
transaction.transaction_id,
transaction.operation_data
)
future_to_participant[future] = participant_name
# 收集响应
for future in as_completed(future_to_participant, timeout=self.timeout):
participant_name = future_to_participant[future]
try:
response = future.result()
transaction.votes[participant_name] = response
if response == ThreePCResponse.YES:
yes_votes += 1
self.logger.info(f"参与者 {participant_name} CanCommit: YES")
else:
no_votes += 1
self.logger.warning(f"参与者 {participant_name} CanCommit: {response}")
except Exception as e:
self.logger.error(f"参与者 {participant_name} CanCommit超时: {e}")
no_votes += 1
transaction.votes[participant_name] = ThreePCResponse.TIMEOUT
# 检查投票结果
if no_votes == 0 and yes_votes == len(transaction.participants):
self.logger.info(f"事务 {transaction.transaction_id} CanCommit阶段成功")
return True
else:
self.logger.warning(f"事务 {transaction.transaction_id} CanCommit阶段失败，YES: {yes_votes}, NO: {no_votes}")
return False
def _pre_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""PreCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入PreCommit阶段")
transaction.state = ThreePCState.PRE_COMMIT
ack_count = 0
# 并行发送PreCommit请求
with ThreadPoolExecutor(max_workers=len(transaction.participants)) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
future = executor.submit(
participant.pre_commit,
transaction.transaction_id
)
future_to_participant[future] = participant_name
# 收集响应
for future in as_completed(future_to_participant, timeout=self.timeout):
participant_name = future_to_participant[future]
try:
response = future.result()
transaction.votes[participant_name] = response
if response == ThreePCResponse.ACK:
ack_count += 1
self.logger.info(f"参与者 {participant_name} PreCommit: ACK")
else:
self.logger.warning(f"参与者 {participant_name} PreCommit失败: {response}")
except Exception as e:
self.logger.error(f"参与者 {participant_name} PreCommit超时: {e}")
transaction.votes[participant_name] = ThreePCResponse.TIMEOUT
# 检查PreCommit结果
if ack_count == len(transaction.participants):
self.logger.info(f"事务 {transaction.transaction_id} PreCommit阶段成功")
return True
else:
self.logger.warning(f"事务 {transaction.transaction_id} PreCommit阶段失败，ACK: {ack_count}/{len(transaction.participants)}")
return False
def _do_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""DoCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入DoCommit阶段")
transaction.state = ThreePCState.DO_COMMIT
ack_count = 0
# 并行发送DoCommit请求
with ThreadPoolExecutor(max_workers=len(transaction.participants)) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if not participant:
continue
future = executor.submit(
participant.do_commit,
transaction.transaction_id
)
future_to_participant[future] = participant_name
# 收集响应
for future in as_completed(future_to_participant, timeout=self.timeout):
participant_name = future_to_participant[future]
try:
response = future.result()
transaction.acknowledgements[participant_name] = response
if response == ThreePCResponse.ACK:
ack_count += 1
self.logger.info(f"参与者 {participant_name} DoCommit: ACK")
else:
self.logger.error(f"参与者 {participant_name} DoCommit失败: {response}")
except Exception as e:
self.logger.error(f"参与者 {participant_name} DoCommit超时: {e}")
transaction.acknowledgements[participant_name] = ThreePCResponse.TIMEOUT
# 检查最终结果
if ack_count == len(transaction.participants):
transaction.state = ThreePCState.COMMITTED
self.logger.info(f"事务 {transaction.transaction_id} 提交完成")
return True
else:
transaction.state = ThreePCState.ABORTED
self.logger.error(f"事务 {transaction.transaction_id} 提交失败，成功: {ack_count}/{len(transaction.participants)}")
return False
def _abort_transaction(self, transaction: ThreePCTransaction):
"""中止事务"""
self.logger.info(f"中止事务: {transaction.transaction_id}")
transaction.state = ThreePCState.ABORTED
# 通知所有参与者中止事务
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if participant:
try:
participant.abort_transaction(transaction.transaction_id)
self.logger.info(f"参与者 {participant_name} 事务中止成功")
except Exception as e:
self.logger.error(f"参与者 {participant_name} 事务中止失败: {e}")
# 使用示例
def basic_3pc_example():
"""基础三阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建协调者
coordinator = ThreePCCoordinator(timeout=10, max_retries=3)
# 创建参与者
participants = [
ThreePCParticipant("BankServer1", timeout=8),
ThreePCParticipant("BankServer2", timeout=8),
ThreePCParticipant("InventoryServer", timeout=8)
]
# 注册参与者
for participant in participants:
coordinator.add_participant(participant)
# 执行分布式转账事务
transaction_id = "3PC_TXN_001"
participant_names = ["BankServer1", "BankServer2", "InventoryServer"]
operation_data = {
"transfer_amount": 5000.0,
"from_account": "business_user",
"to_account": "supplier_user",
"inventory_changes": ["item_A", "item_B"]
}
# 执行事务
success = coordinator.begin_transaction(transaction_id, participant_names, operation_data)
if success:
print(f"三阶段提交事务 {transaction_id} 执行成功")
else:
print(f"三阶段提交事务 {transaction_id} 执行失败")
---
b.故障恢复机制
---
# 带故障恢复的三阶段提交
import pickle
import os
from pathlib import Path
from typing import Dict, List, Optional, Set
class ThreePCStateRecovery:
"""三阶段提交状态恢复管理"""
def __init__(self, recovery_path: str = "./3pc_recovery"):
self.recovery_path = Path(recovery_path)
self.recovery_path.mkdir(exist_ok=True)
self.logger = logging.getLogger("ThreePCRecovery")
def save_coordinator_state(self, transactions: Dict[str, ThreePCTransaction]):
"""保存协调者状态"""
try:
state_file = self.recovery_path / "coordinator_state.pkl"
with open(state_file, 'wb') as f:
pickle.dump(transactions, f)
self.logger.info("协调者状态已保存")
except Exception as e:
self.logger.error(f"保存协调者状态失败: {e}")
def load_coordinator_state(self) -> Dict[str, ThreePCTransaction]:
"""加载协调者状态"""
state_file = self.recovery_path / "coordinator_state.pkl"
if not state_file.exists():
return {}
try:
with open(state_file, 'rb') as f:
transactions = pickle.load(f)
self.logger.info("协调者状态已加载")
return transactions
except Exception as e:
self.logger.error(f"加载协调者状态失败: {e}")
return {}
def get_inconsistent_transactions(self, transactions: Dict[str, ThreePCTransaction]) -> List[str]:
"""获取可能不一致的事务"""
inconsistent = []
current_time = time.time()
for transaction_id, transaction in transactions.items():
# 检查超时的事务
if transaction.state in [ThreePCState.CAN_COMMIT, ThreePCState.PRE_COMMIT, ThreePCState.DO_COMMIT]:
if current_time - transaction.created_at > transaction.timeout:
inconsistent.append(transaction_id)
return inconsistent
class FaultTolerantThreePCCoordinator(ThreePCCoordinator):
"""容错三阶段提交协调者"""
def __init__(self, timeout: int = 10, max_retries: int = 3, recovery_path: str = "./3pc_recovery"):
super().__init__(timeout, max_retries)
self.recovery = ThreePCStateRecovery(recovery_path)
self.last_save_time = 0
self.save_interval = 5 # 每5秒保存一次状态
def begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""开始容错三阶段提交事务"""
# 加载之前的状态
self._load_state()
# 检查是否有未完成的相同事务
if transaction_id in self.transactions:
self.logger.warning(f"发现未完成的事务 {transaction_id}，状态: {self.transactions[transaction_id].state}")
# 尝试恢复或中止旧事务
self._handle_inconsistent_transaction(transaction_id)
success = super().begin_transaction(transaction_id, participants, operation_data)
# 保存状态
self._save_state()
return success
def _can_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""重写CanCommit阶段，添加状态保存"""
result = super()._can_commit_phase(transaction)
self._save_state() # 保存阶段状态
return result
def _pre_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""重写PreCommit阶段，添加状态保存"""
result = super()._pre_commit_phase(transaction)
self._save_state() # 保存阶段状态
return result
def _do_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""重写DoCommit阶段，添加状态保存"""
result = super()._do_commit_phase(transaction)
if result:
# 成功后清理状态
del self.transactions[transaction.transaction_id]
self._save_state() # 保存最终状态
return result
def _load_state(self):
"""加载协调者状态"""
try:
loaded_transactions = self.recovery.load_coordinator_state()
if loaded_transactions:
self.transactions.update(loaded_transactions)
self.logger.info(f"加载了 {len(loaded_transactions)} 个未完成事务")
except Exception as e:
self.logger.error(f"加载状态失败: {e}")
def _save_state(self):
"""保存协调者状态"""
current_time = time.time()
if current_time - self.last_save_time > self.save_interval:
try:
self.recovery.save_coordinator_state(self.transactions)
self.last_save_time = current_time
except Exception as e:
self.logger.error(f"保存状态失败: {e}")
def _handle_inconsistent_transaction(self, transaction_id: str):
"""处理不一致的事务"""
transaction = self.transactions.get(transaction_id)
if not transaction:
return
self.logger.warning(f"处理不一致事务 {transaction_id}, 状态: {transaction.state}")
try:
if transaction.state == ThreePCState.CAN_COMMIT:
# CanCommit阶段，直接中止
self._abort_transaction(transaction)
del self.transactions[transaction_id]
elif transaction.state == ThreePCState.PRE_COMMIT:
# PreCommit阶段，尝试与参与者通信决定
if self._participant_consensus(transaction):
# 参与者共识可以继续，进入DoCommit
self._do_commit_phase(transaction)
else:
# 共识失败，中止事务
self._abort_transaction(transaction)
del self.transactions[transaction_id]
elif transaction.state == ThreePCState.DO_COMMIT:
# DoCommit阶段，需要继续执行直到完成
self._do_commit_phase(transaction)
except Exception as e:
self.logger.error(f"处理不一致事务 {transaction_id} 失败: {e}")
self._abort_transaction(transaction)
if transaction_id in self.transactions:
del self.transactions[transaction_id]
def _participant_consensus(self, transaction: ThreePCTransaction) -> bool:
"""与参与者达成共识"""
ready_participants = 0
total_participants = len(transaction.participants)
# 询问所有参与者当前状态
with ThreadPoolExecutor(max_workers=total_participants) as executor:
future_to_participant = {}
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if participant:
future = executor.submit(
self._check_participant_state,
participant, transaction.transaction_id
)
future_to_participant[future] = participant_name
for future in as_completed(future_to_participant, timeout=5):
try:
can_continue = future.result()
if can_continue:
ready_participants += 1
except Exception as e:
self.logger.error(f"检查参与者状态失败: {e}")
# 如果超过一半的参与者准备就绪，可以继续
return ready_participants > total_participants // 2
def _check_participant_state(self, participant: ThreePCParticipant, transaction_id: str) -> bool:
"""检查参与者状态"""
try:
# 检查参与者是否在PreCommit状态
return hasattr(participant, 'local_transactions') and transaction_id in participant.local_transactions
except Exception:
return False
def recover_transactions(self):
"""恢复未完成的事务"""
self.logger.info("开始恢复未完成的三阶段提交事务")
# 使用示例
def fault_tolerant_3pc_example():
"""容错三阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建容错协调者
coordinator = FaultTolerantThreePCCoordinator(
timeout=15,
max_retries=3,
recovery_path="./3pc_recovery"
)
# 创建参与者
participants = [
ThreePCParticipant("OrderService", timeout=12),
ThreePCParticipant("PaymentService", timeout=12),
ThreePCParticipant("ShippingService", timeout=12)
]
for participant in participants:
coordinator.add_participant(participant)
# 执行电商订单事务
transaction_id = "ECOM_ORDER_001"
participant_names = ["OrderService", "PaymentService", "ShippingService"]
operation_data = {
"order_id": "ORD12345",
"user_id": "CUST678",
"amount": 1599.99,
"items": ["laptop", "mouse"],
"shipping_address": "北京市朝阳区"
}
try:
success = coordinator.begin_transaction(transaction_id, participant_names, operation_data)
if success:
print(f"电商订单事务 {transaction_id} 执行成功")
else:
print(f"电商订单事务 {transaction_id} 执行失败")
except KeyboardInterrupt:
print("模拟协调者崩溃...")
# 模拟协调者重启
print("协调者重启，恢复未完成的事务...")
new_coordinator = FaultTolerantThreePCCoordinator(
timeout=15,
max_retries=3,
recovery_path="./3pc_recovery"
)
for participant in participants:
new_coordinator.add_participant(participant)
new_coordinator.recover_transactions()
---
c.优化实现
---
# 优化的三阶段提交实现
import asyncio
import random
from typing import Dict, List, Optional, Set, Coroutine
from concurrent.futures import ThreadPoolExecutor
class AsyncThreePCParticipant(ThreePCParticipant):
"""异步三阶段提交参与者"""
def __init__(self, name: str, timeout: int = 5, failure_rate: float = 0.0):
super().__init__(name, timeout)
self.failure_rate = failure_rate
self.async_executor = ThreadPoolExecutor(max_workers=4)
async def async_can_commit(self, transaction_id: str, operation_data: Dict) -> ThreePCResponse:
"""异步CanCommit阶段"""
loop = asyncio.get_event_loop()
# 模拟随机失败
if random.random() < self.failure_rate:
self.logger.warning(f"参与者 {self.name} 模拟失败")
return ThreePCResponse.NO
try:
response = await loop.run_in_executor(
self.async_executor,
self.can_commit,
transaction_id,
operation_data
)
return response
except Exception as e:
self.logger.error(f"异步CanCommit异常: {e}")
return ThreePCResponse.NO
async def async_pre_commit(self, transaction_id: str) -> ThreePCResponse:
"""异步PreCommit阶段"""
loop = asyncio.get_event_loop()
try:
response = await loop.run_in_executor(
self.async_executor,
self.pre_commit,
transaction_id
)
return response
except Exception as e:
self.logger.error(f"异步PreCommit异常: {e}")
return ThreePCResponse.NO
async def async_do_commit(self, transaction_id: str) -> ThreePCResponse:
"""异步DoCommit阶段"""
loop = asyncio.get_event_loop()
try:
response = await loop.run_in_executor(
self.async_executor,
self.do_commit,
transaction_id
)
return response
except Exception as e:
self.logger.error(f"异步DoCommit异常: {e}")
return ThreePCResponse.NO
class OptimizedThreePCCoordinator(ThreePCCoordinator):
"""优化的三阶段提交协调者"""
def __init__(self, timeout: int = 10, max_retries: int = 3, batch_size: int = 5):
super().__init__(timeout, max_retries)
self.batch_size = batch_size
self.performance_metrics = {
'total_transactions': 0,
'successful_transactions': 0,
'failed_transactions': 0,
'average_duration': 0.0
}
async def async_begin_transaction(self, transaction_id: str, participants: List[str], operation_data: Dict) -> bool:
"""异步开始三阶段提交事务"""
start_time = time.time()
self.logger.info(f"开始异步三阶段提交事务: {transaction_id}")
# 更新性能指标
self.performance_metrics['total_transactions'] += 1
transaction = ThreePCTransaction(
transaction_id=transaction_id,
participants=participants,
operation_data=operation_data,
timeout=self.timeout
)
self.transactions[transaction_id] = transaction
try:
# 第一阶段：CanCommit
if not await self._async_can_commit_phase(transaction):
await self._async_abort_transaction(transaction)
self._update_performance_metrics(False, start_time)
return False
# 第二阶段：PreCommit
if not await self._async_pre_commit_phase(transaction):
await self._async_abort_transaction(transaction)
self._update_performance_metrics(False, start_time)
return False
# 第三阶段：DoCommit
success = await self._async_do_commit_phase(transaction)
self._update_performance_metrics(success, start_time)
return success
except Exception as e:
self.logger.error(f"异步事务 {transaction_id} 执行异常: {e}")
await self._async_abort_transaction(transaction)
self._update_performance_metrics(False, start_time)
return False
async def _async_can_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""异步CanCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入异步CanCommit阶段")
transaction.state = ThreePCState.CAN_COMMIT
# 分批处理参与者
for i in range(0, len(transaction.participants), self.batch_size):
batch_participants = transaction.participants[i:i + self.batch_size]
# 创建异步任务
tasks = []
for participant_name in batch_participants:
participant = self.participants.get(participant_name)
if isinstance(participant, AsyncThreePCParticipant):
task = participant.async_can_commit(transaction.transaction_id, transaction.operation_data)
tasks.append((participant_name, task))
else:
# 同步参与者适配
loop = asyncio.get_event_loop()
task = loop.run_in_executor(
None,
participant.can_commit,
transaction.transaction_id,
transaction.operation_data
)
tasks.append((participant_name, task))
# 并行执行
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
# 处理结果
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"参与者 {participant_name} CanCommit异常: {result}")
transaction.votes[participant_name] = ThreePCResponse.TIMEOUT
return False
elif result == ThreePCResponse.YES:
transaction.votes[participant_name] = result
self.logger.info(f"参与者 {participant_name} CanCommit: YES")
else:
self.logger.warning(f"参与者 {participant_name} CanCommit: {result}")
return False
self.logger.info(f"事务 {transaction.transaction_id} 异步CanCommit阶段成功")
return True
async def _async_pre_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""异步PreCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入异步PreCommit阶段")
transaction.state = ThreePCState.PRE_COMMIT
# 分批处理
for i in range(0, len(transaction.participants), self.batch_size):
batch_participants = transaction.participants[i:i + self.batch_size]
tasks = []
for participant_name in batch_participants:
participant = self.participants.get(participant_name)
if isinstance(participant, AsyncThreePCParticipant):
task = participant.async_pre_commit(transaction.transaction_id)
tasks.append((participant_name, task))
else:
loop = asyncio.get_event_loop()
task = loop.run_in_executor(
None,
participant.pre_commit,
transaction.transaction_id
)
tasks.append((participant_name, task))
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"参与者 {participant_name} PreCommit异常: {result}")
return False
elif result == ThreePCResponse.ACK:
self.logger.info(f"参与者 {participant_name} PreCommit: ACK")
else:
self.logger.warning(f"参与者 {participant_name} PreCommit失败: {result}")
return False
self.logger.info(f"事务 {transaction.transaction_id} 异步PreCommit阶段成功")
return True
async def _async_do_commit_phase(self, transaction: ThreePCTransaction) -> bool:
"""异步DoCommit阶段"""
self.logger.info(f"事务 {transaction.transaction_id} 进入异步DoCommit阶段")
transaction.state = ThreePCState.DO_COMMIT
ack_count = 0
tasks = []
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if isinstance(participant, AsyncThreePCParticipant):
task = participant.async_do_commit(transaction.transaction_id)
tasks.append((participant_name, task))
else:
loop = asyncio.get_event_loop()
task = loop.run_in_executor(
None,
participant.do_commit,
transaction.transaction_id
)
tasks.append((participant_name, task))
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"参与者 {participant_name} DoCommit异常: {result}")
elif result == ThreePCResponse.ACK:
ack_count += 1
self.logger.info(f"参与者 {participant_name} DoCommit: ACK")
else:
self.logger.error(f"参与者 {participant_name} DoCommit失败: {result}")
# 检查结果
if ack_count == len(transaction.participants):
transaction.state = ThreePCState.COMMITTED
self.logger.info(f"事务 {transaction.transaction_id} 异步提交完成")
return True
else:
transaction.state = ThreePCState.ABORTED
self.logger.error(f"事务 {transaction.transaction_id} 异步提交失败，成功: {ack_count}/{len(transaction.participants)}")
return False
async def _async_abort_transaction(self, transaction: ThreePCTransaction):
"""异步中止事务"""
self.logger.info(f"异步中止事务: {transaction.transaction_id}")
transaction.state = ThreePCState.ABORTED
tasks = []
for participant_name in transaction.participants:
participant = self.participants.get(participant_name)
if participant:
loop = asyncio.get_event_loop()
task = loop.run_in_executor(
None,
participant.abort_transaction,
transaction.transaction_id
)
tasks.append(task)
await asyncio.gather(*tasks, return_exceptions=True)
def _update_performance_metrics(self, success: bool, start_time: float):
"""更新性能指标"""
duration = time.time() - start_time
if success:
self.performance_metrics['successful_transactions'] += 1
else:
self.performance_metrics['failed_transactions'] += 1
# 更新平均执行时间
total = self.performance_metrics['total_transactions']
current_avg = self.performance_metrics['average_duration']
self.performance_metrics['average_duration'] = (current_avg * (total - 1) + duration) / total
self.logger.info(f"事务执行时间: {duration:.3f}s, 成功率: {self.performance_metrics['successful_transactions']}/{total}")
def get_performance_report(self) -> Dict:
"""获取性能报告"""
return {
**self.performance_metrics,
'success_rate': self.performance_metrics['successful_transactions'] / max(1, self.performance_metrics['total_transactions']),
'average_duration': self.performance_metrics['average_duration']
}
# 使用示例
async def optimized_3pc_example():
"""优化三阶段提交示例"""
logging.basicConfig(level=logging.INFO)
# 创建优化的协调者
coordinator = OptimizedThreePCCoordinator(timeout=10, max_retries=3, batch_size=3)
# 创建异步参与者
participants = [
AsyncThreePCParticipant("AsyncService1", timeout=8, failure_rate=0.1),
AsyncThreePCParticipant("AsyncService2", timeout=8, failure_rate=0.1),
AsyncThreePCParticipant("AsyncService3", timeout=8, failure_rate=0.1),
AsyncThreePCParticipant("AsyncService4", timeout=8, failure_rate=0.1),
AsyncThreePCParticipant("AsyncService5", timeout=8, failure_rate=0.1)
]
for participant in participants:
coordinator.add_participant(participant)
# 执行多个并发事务
tasks = []
for i in range(5):
transaction_id = f"ASYNC_TXN_{i:03d}"
participant_names = [f"AsyncService{j}" for j in range(1, 4)] # 使用前3个服务
operation_data = {
"batch_id": i,
"items": [f"item_{i}_{j}" for j in range(5)],
"priority": random.choice(["high", "medium", "low"])
}
task = coordinator.async_begin_transaction(transaction_id, participant_names, operation_data)
tasks.append((transaction_id, task))
# 等待所有事务完成
results = []
for transaction_id, task in tasks:
try:
success = await asyncio.wait_for(task, timeout=30)
results.append((transaction_id, success))
except asyncio.TimeoutError:
results.append((transaction_id, False))
# 输出结果
for transaction_id, success in results:
print(f"事务 {transaction_id}: {'成功' if success else '失败'}")
# 性能报告
report = coordinator.get_performance_report()
print(f"性能报告: {report}")
if __name__ == "__main__":
# 运行异步示例
asyncio.run(optimized_3pc_example())
---
06.性能优化
a.异步并发
使用异步IO和并发处理提高系统吞吐量，减少事务等待时间。合理设置并发批次大小，平衡性能和资源消耗。
b.批量处理
对参与者请求进行批量处理，减少网络往返次数和系统开销。使用分批策略避免过载，确保系统稳定性。
c.智能超时
根据网络状况和负载情况动态调整超时时间，避免因超时设置不合理导致的不必要重试或等待。
6.3 TCC模式
01.TCC模式概述
a.基本概念
TCC（Try-Confirm-Cancel）是一种补偿型分布式事务解决方案，将业务操作分为Try预留资源、Confirm确认执行、Cancel撤销操作三个阶段。每个阶段都由业务方实现，提供更灵活的事务控制。
b.模式特点
与传统数据库事务不同，TCC模式在应用层实现事务控制，将事务逻辑嵌入业务代码中。通过预留资源和补偿机制，在分布式环境下保证最终一致性。
c.适用场景
适用于业务逻辑复杂、需要高性能分布式事务的场景，如电商订单处理、金融支付、物流配送等对性能和一致性要求较高的业务。
02.Try阶段
a.资源预留
Try阶段完成所有业务检查并预留业务资源，但不执行最终的业务操作。检查账户余额、库存数量、用户权限等业务条件，确保后续操作能够成功执行。
b.状态设置
为每个参与服务设置中间状态，标记资源已被预留。通过数据库字段或缓存状态记录预留信息，防止其他事务重复预留相同资源。
c.超时处理
Try阶段设置合理的超时时间，防止资源被长时间锁定。如果后续Confirm或Cancel未及时执行，系统需要自动释放预留资源。
03.Confirm阶段
a.确认执行
Confirm阶段在所有服务Try成功后执行真正的业务操作，如资金转移、库存扣减、订单创建等。由于Try阶段已预留资源，Confirm操作通常能够快速执行。
b.幂等性保证
Confirm操作必须保证幂等性，即重复执行相同操作不会产生副作用。通过检查操作状态或使用唯一事务ID来避免重复执行。
c.异常处理
Confirm阶段如果发生异常，需要根据具体情况进行补偿处理或告警通知，确保系统能够达到一致状态。
04.Cancel阶段
a.资源释放
Cancel阶段释放Try阶段预留的资源，取消业务操作。根据业务类型执行不同的补偿逻辑，如恢复库存、退款、取消订单等。
b.补偿机制
补偿操作必须是原业务操作的逆操作，能够将系统状态恢复到事务执行前的状态。补偿逻辑需要考虑各种边界情况和异常场景。
c.部分失败处理
当部分服务Confirm成功、部分失败时，需要通过Cancel操作回滚已成功的操作，确保数据一致性。
05.实现示例
a.基础TCC框架
---
# 基础TCC模式实现
import logging
import time
import uuid
from typing import Dict, List, Optional, Any, Callable
from enum import Enum
from dataclasses import dataclass
from abc import ABC, abstractmethod
class TCCStatus(Enum):
"""TCC事务状态"""
INIT = "INIT"
TRYING = "TRYING"
CONFIRMING = "CONFIRMING"
CANCELLING = "CANCELLING"
CONFIRMED = "CONFIRMED"
CANCELLED = "CANCELLED"
TIMEOUT = "TIMEOUT"
class TCCParticipant(ABC):
"""TCC参与者抽象基类"""
def __init__(self, name: str):
self.name = name
self.logger = logging.getLogger(f"TCC-{name}")
@abstractmethod
def try_operation(self, transaction_id: str, params: Dict) -> bool:
"""Try阶段：预留资源"""
pass
@abstractmethod
def confirm_operation(self, transaction_id: str, params: Dict) -> bool:
"""Confirm阶段：确认执行"""
pass
@abstractmethod
def cancel_operation(self, transaction_id: str, params: Dict) -> bool:
"""Cancel阶段：取消操作"""
pass
class BankingTCCService(TCCParticipant):
"""银行服务TCC实现"""
def __init__(self, name: str, accounts: Dict[str, float]):
super().__init__(name)
self.accounts = accounts.copy()
self.frozen_amounts = {} # 冻结金额
self.transactions = {} # 事务记录
def try_operation(self, transaction_id: str, params: Dict) -> bool:
"""Try阶段：检查余额并冻结资金"""
account_id = params['account_id']
amount = params['amount']
operation_type = params.get('operation_type', 'debit') # debit/credit
self.logger.info(f"Try操作: {account_id} {operation_type} {amount}")
try:
if operation_type == 'debit':
# 扣款操作：检查余额是否充足
if account_id not in self.accounts:
self.logger.error(f"账户不存在: {account_id}")
return False
available_balance = self.accounts[account_id] - self.frozen_amounts.get(account_id, 0)
if available_balance < amount:
self.logger.error(f"余额不足: 可用{available_balance}, 需要{amount}")
return False
# 冻结资金
if transaction_id not in self.frozen_amounts:
self.frozen_amounts[transaction_id] = 0
self.frozen_amounts[transaction_id] += amount
# 记录事务
self.transactions[transaction_id] = {
'account_id': account_id,
'amount': amount,
'operation_type': operation_type,
'status': TCCStatus.TRYING,
'created_at': time.time()
}
elif operation_type == 'credit':
# 存款操作：总是成功
self.transactions[transaction_id] = {
'account_id': account_id,
'amount': amount,
'operation_type': operation_type,
'status': TCCStatus.TRYING,
'created_at': time.time()
}
self.logger.info(f"Try成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Try操作异常: {e}")
return False
def confirm_operation(self, transaction_id: str, params: Dict) -> bool:
"""Confirm阶段：执行实际操作"""
if transaction_id not in self.transactions:
self.logger.warning(f"事务记录不存在: {transaction_id}")
return False
transaction = self.transactions[transaction_id]
account_id = transaction['account_id']
amount = transaction['amount']
operation_type = transaction['operation_type']
self.logger.info(f"Confirm操作: {account_id} {operation_type} {amount}")
try:
if operation_type == 'debit':
# 执行扣款
if account_id in self.accounts:
self.accounts[account_id] -= amount
self.logger.info(f"扣款成功: {account_id} -{amount}")
# 释放冻结资金
if transaction_id in self.frozen_amounts:
del self.frozen_amounts[transaction_id]
elif operation_type == 'credit':
# 执行存款
if account_id in self.accounts:
self.accounts[account_id] += amount
self.logger.info(f"存款成功: {account_id} +{amount}")
# 更新事务状态
transaction['status'] = TCCStatus.CONFIRMED
transaction['confirmed_at'] = time.time()
self.logger.info(f"Confirm成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Confirm操作异常: {e}")
return False
def cancel_operation(self, transaction_id: str, params: Dict) -> bool:
"""Cancel阶段：释放冻结资金"""
if transaction_id not in self.transactions:
self.logger.warning(f"事务记录不存在: {transaction_id}")
return False
transaction = self.transactions[transaction_id]
account_id = transaction['account_id']
amount = transaction['amount']
operation_type = transaction['operation_type']
self.logger.info(f"Cancel操作: {account_id} {operation_type} {amount}")
try:
if operation_type == 'debit':
# 释放冻结资金
if transaction_id in self.frozen_amounts:
del self.frozen_amounts[transaction_id]
self.logger.info(f"释放冻结资金: {account_id} +{amount}")
elif operation_type == 'credit':
# 存款操作无需补偿
pass
# 更新事务状态
transaction['status'] = TCCStatus.CANCELLED
transaction['cancelled_at'] = time.time()
self.logger.info(f"Cancel成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Cancel操作异常: {e}")
return False
def get_balance(self, account_id: str) -> float:
"""获取账户余额"""
return self.accounts.get(account_id, 0.0)
def get_frozen_amount(self, account_id: str) -> float:
"""获取冻结金额"""
total_frozen = 0
for tx_id, frozen_amount in self.frozen_amounts.items():
if tx_id in self.transactions and self.transactions[tx_id]['account_id'] == account_id:
total_frozen += frozen_amount
return total_frozen
class InventoryTCCService(TCCParticipant):
"""库存服务TCC实现"""
def __init__(self, name: str, inventory: Dict[str, int]):
super().__init__(name)
self.inventory = inventory.copy()
self.reserved_inventory = {} # 预留库存
self.transactions = {}
def try_operation(self, transaction_id: str, params: Dict) -> bool:
"""Try阶段：检查库存并预留"""
product_id = params['product_id']
quantity = params['quantity']
self.logger.info(f"Try操作: {product_id} 预留 {quantity}")
try:
# 检查库存是否充足
available_inventory = self.inventory.get(product_id, 0)
reserved_quantity = self.reserved_inventory.get(product_id, 0)
actual_available = available_inventory - reserved_quantity
if actual_available < quantity:
self.logger.error(f"库存不足: 可用{actual_available}, 需要{quantity}")
return False
# 预留库存
if transaction_id not in self.reserved_inventory:
self.reserved_inventory[transaction_id] = {}
if product_id not in self.reserved_inventory[transaction_id]:
self.reserved_inventory[transaction_id][product_id] = 0
self.reserved_inventory[transaction_id][product_id] += quantity
# 记录事务
self.transactions[transaction_id] = {
'product_id': product_id,
'quantity': quantity,
'status': TCCStatus.TRYING,
'created_at': time.time()
}
self.logger.info(f"Try成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Try操作异常: {e}")
return False
def confirm_operation(self, transaction_id: str, params: Dict) -> bool:
"""Confirm阶段：扣减库存"""
if transaction_id not in self.transactions:
self.logger.warning(f"事务记录不存在: {transaction_id}")
return False
transaction = self.transactions[transaction_id]
product_id = transaction['product_id']
quantity = transaction['quantity']
self.logger.info(f"Confirm操作: {product_id} 扣减 {quantity}")
try:
# 扣减库存
if product_id in self.inventory:
self.inventory[product_id] -= quantity
self.logger.info(f"库存扣减成功: {product_id} -{quantity}")
# 释放预留
if transaction_id in self.reserved_inventory:
if product_id in self.reserved_inventory[transaction_id]:
del self.reserved_inventory[transaction_id][product_id]
if not self.reserved_inventory[transaction_id]:
del self.reserved_inventory[transaction_id]
# 更新事务状态
transaction['status'] = TCCStatus.CONFIRMED
transaction['confirmed_at'] = time.time()
self.logger.info(f"Confirm成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Confirm操作异常: {e}")
return False
def cancel_operation(self, transaction_id: str, params: Dict) -> bool:
"""Cancel阶段：释放预留库存"""
if transaction_id not in self.transactions:
self.logger.warning(f"事务记录不存在: {transaction_id}")
return False
transaction = self.transactions[transaction_id]
product_id = transaction['product_id']
quantity = transaction['quantity']
self.logger.info(f"Cancel操作: {product_id} 释放 {quantity}")
try:
# 释放预留库存
if transaction_id in self.reserved_inventory:
if product_id in self.reserved_inventory[transaction_id]:
del self.reserved_inventory[transaction_id][product_id]
if not self.reserved_inventory[transaction_id]:
del self.reserved_inventory[transaction_id]
# 更新事务状态
transaction['status'] = TCCStatus.CANCELLED
transaction['cancelled_at'] = time.time()
self.logger.info(f"Cancel成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"Cancel操作异常: {e}")
return False
def get_inventory(self, product_id: str) -> int:
"""获取库存数量"""
return self.inventory.get(product_id, 0)
class TCCTransactionManager:
"""TCC事务管理器"""
def __init__(self, timeout: int = 30):
self.participants = {}
self.transactions = {}
self.timeout = timeout
self.logger = logging.getLogger("TCCTransactionManager")
def register_participant(self, participant: TCCParticipant):
"""注册参与者"""
self.participants[participant.name] = participant
self.logger.info(f"注册参与者: {participant.name}")
def execute_transaction(self, participants: List[str], operations: List[Dict]) -> bool:
"""执行TCC事务"""
transaction_id = str(uuid.uuid4())
self.logger.info(f"开始TCC事务: {transaction_id}")
try:
# Try阶段
if not self._try_phase(transaction_id, participants, operations):
self.logger.error(f"Try阶段失败: {transaction_id}")
self._cancel_phase(transaction_id, participants, operations)
return False
# Confirm阶段
if not self._confirm_phase(transaction_id, participants, operations):
self.logger.error(f"Confirm阶段失败: {transaction_id}")
return False
self.logger.info(f"TCC事务成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"TCC事务异常: {transaction_id}, {e}")
self._cancel_phase(transaction_id, participants, operations)
return False
def _try_phase(self, transaction_id: str, participants: List[str], operations: List[Dict]) -> bool:
"""Try阶段"""
self.logger.info(f"Try阶段开始: {transaction_id}")
success_count = 0
for i, (participant_name, operation) in enumerate(zip(participants, operations)):
participant = self.participants.get(participant_name)
if not participant:
self.logger.error(f"参与者不存在: {participant_name}")
return False
try:
if participant.try_operation(transaction_id, operation):
success_count += 1
self.logger.info(f"Try成功: {participant_name}")
else:
self.logger.error(f"Try失败: {participant_name}")
return False
except Exception as e:
self.logger.error(f"Try异常: {participant_name}, {e}")
return False
self.logger.info(f"Try阶段成功: {success_count}/{len(participants)}")
return True
def _confirm_phase(self, transaction_id: str, participants: List[str], operations: List[Dict]) -> bool:
"""Confirm阶段"""
self.logger.info(f"Confirm阶段开始: {transaction_id}")
success_count = 0
for participant_name, operation in zip(participants, operations):
participant = self.participants.get(participant_name)
if not participant:
continue
try:
if participant.confirm_operation(transaction_id, operation):
success_count += 1
self.logger.info(f"Confirm成功: {participant_name}")
else:
self.logger.error(f"Confirm失败: {participant_name}")
# Confirm失败通常不会重试，因为Try已成功
except Exception as e:
self.logger.error(f"Confirm异常: {participant_name}, {e}")
return success_count == len(participants)
def _cancel_phase(self, transaction_id: str, participants: List[str], operations: List[Dict]):
"""Cancel阶段"""
self.logger.info(f"Cancel阶段开始: {transaction_id}")
for participant_name, operation in zip(participants, operations):
participant = self.participants.get(participant_name)
if not participant:
continue
try:
if participant.cancel_operation(transaction_id, operation):
self.logger.info(f"Cancel成功: {participant_name}")
else:
self.logger.error(f"Cancel失败: {participant_name}")
except Exception as e:
self.logger.error(f"Cancel异常: {participant_name}, {e}")
# 使用示例
def basic_tcc_example():
"""基础TCC模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建服务实例
banking_service = BankingTCCService("BankService", {
"user001": 10000.0,
"merchant001": 5000.0
})
inventory_service = InventoryTCCService("InventoryService", {
"laptop": 100,
"phone": 200
})
# 创建事务管理器
transaction_manager = TCCTransactionManager(timeout=30)
transaction_manager.register_participant(banking_service)
transaction_manager.register_participant(inventory_service)
print(f"转账前余额 - 用户: {banking_service.get_balance('user001')}, 商家: {banking_service.get_balance('merchant001')}")
print(f"扣减前库存 - 笔记本: {inventory_service.get_inventory('laptop')}")
# 执行购物事务
participants = ["BankService", "InventoryService"]
operations = [
{
"account_id": "user001",
"amount": 5000.0,
"operation_type": "debit"
},
{
"product_id": "laptop",
"quantity": 2
}
]
success = transaction_manager.execute_transaction(participants, operations)
if success:
print("购物事务执行成功")
print(f"转账后余额 - 用户: {banking_service.get_balance('user001')}, 商家: {banking_service.get_balance('merchant001')}")
print(f"扣减后库存 - 笔记本: {inventory_service.get_inventory('laptop')}")
else:
print("购物事务执行失败")
print(f"失败后余额 - 用户: {banking_service.get_balance('user001')}, 商家: {banking_service.get_balance('merchant001')}")
print(f"失败后库存 - 笔记本: {inventory_service.get_inventory('laptop')}")
---
b.分布式TCC协调器
---
# 分布式TCC协调器实现
import asyncio
import aiohttp
from typing import Dict, List, Optional, Any
from dataclasses import dataclass, field
import json
@dataclass
class TCCTransaction:
"""TCC事务数据结构"""
transaction_id: str
participants: List[str]
operations: List[Dict]
status: TCCStatus = TCCStatus.INIT
created_at: float = field(default_factory=time.time)
timeout: int = 30
results: Dict[str, Any] = field(default_factory=dict)
class DistributedTCCTransactionManager:
"""分布式TCC事务管理器"""
def __init__(self, timeout: int = 30, retry_count: int = 3):
self.participants = {}
self.transactions = {}
self.timeout = timeout
self.retry_count = retry_count
self.logger = logging.getLogger("DistributedTCCTransactionManager")
async def register_participant(self, name: str, endpoint: str, health_check: str = None):
"""注册分布式参与者"""
self.participants[name] = {
'endpoint': endpoint,
'health_check': health_check or f"{endpoint}/health",
'status': 'active'
}
self.logger.info(f"注册分布式参与者: {name} - {endpoint}")
async def execute_transaction(self, participants: List[str], operations: List[Dict]) -> bool:
"""执行分布式TCC事务"""
transaction_id = str(uuid.uuid4())
transaction = TCCTransaction(
transaction_id=transaction_id,
participants=participants,
operations=operations,
timeout=self.timeout
)
self.transactions[transaction_id] = transaction
self.logger.info(f"开始分布式TCC事务: {transaction_id}")
try:
# Try阶段
if not await self._distributed_try_phase(transaction):
self.logger.error(f"分布式Try阶段失败: {transaction_id}")
await self._distributed_cancel_phase(transaction)
return False
# Confirm阶段
if not await self._distributed_confirm_phase(transaction):
self.logger.error(f"分布式Confirm阶段失败: {transaction_id}")
return False
self.logger.info(f"分布式TCC事务成功: {transaction_id}")
return True
except Exception as e:
self.logger.error(f"分布式TCC事务异常: {transaction_id}, {e}")
await self._distributed_cancel_phase(transaction)
return False
async def _distributed_try_phase(self, transaction: TCCTransaction) -> bool:
"""分布式Try阶段"""
self.logger.info(f"分布式Try阶段开始: {transaction.transaction_id}")
transaction.status = TCCStatus.TRYING
tasks = []
for participant_name, operation in zip(transaction.participants, transaction.operations):
task = self._call_participant_try(
participant_name,
transaction.transaction_id,
operation
)
tasks.append((participant_name, task))
# 并行执行Try操作
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
success_count = 0
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"分布式Try异常: {participant_name}, {result}")
transaction.results[participant_name] = {'success': False, 'error': str(result)}
elif result:
success_count += 1
transaction.results[participant_name] = {'success': True}
self.logger.info(f"分布式Try成功: {participant_name}")
else:
transaction.results[participant_name] = {'success': False}
self.logger.error(f"分布式Try失败: {participant_name}")
return success_count == len(transaction.participants)
async def _distributed_confirm_phase(self, transaction: TCCTransaction) -> bool:
"""分布式Confirm阶段"""
self.logger.info(f"分布式Confirm阶段开始: {transaction.transaction_id}")
transaction.status = TCCStatus.CONFIRMING
tasks = []
for participant_name, operation in zip(transaction.participants, transaction.operations):
task = self._call_participant_confirm(
participant_name,
transaction.transaction_id,
operation
)
tasks.append((participant_name, task))
# 并行执行Confirm操作
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
success_count = 0
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"分布式Confirm异常: {participant_name}, {result}")
transaction.results[participant_name]['confirm_success'] = False
elif result:
success_count += 1
transaction.results[participant_name]['confirm_success'] = True
self.logger.info(f"分布式Confirm成功: {participant_name}")
else:
transaction.results[participant_name]['confirm_success'] = False
self.logger.error(f"分布式Confirm失败: {participant_name}")
return success_count == len(transaction.participants)
async def _distributed_cancel_phase(self, transaction: TCCTransaction):
"""分布式Cancel阶段"""
self.logger.info(f"分布式Cancel阶段开始: {transaction.transaction_id}")
transaction.status = TCCStatus.CANCELLING
tasks = []
for participant_name, operation in zip(transaction.participants, transaction.operations):
task = self._call_participant_cancel(
participant_name,
transaction.transaction_id,
operation
)
tasks.append((participant_name, task))
# 并行执行Cancel操作
results = await asyncio.gather(*[task for _, task in tasks], return_exceptions=True)
for (participant_name, _), result in zip(tasks, results):
if isinstance(result, Exception):
self.logger.error(f"分布式Cancel异常: {participant_name}, {result}")
elif result:
self.logger.info(f"分布式Cancel成功: {participant_name}")
else:
self.logger.error(f"分布式Cancel失败: {participant_name}")
transaction.status = TCCStatus.CANCELLED
async def _call_participant_try(self, participant_name: str, transaction_id: str, params: Dict) -> bool:
"""调用参与者Try方法"""
participant_info = self.participants.get(participant_name)
if not participant_info:
return False
endpoint = participant_info['endpoint']
url = f"{endpoint}/tcc/try"
payload = {
'transaction_id': transaction_id,
'params': params
}
return await self._http_call_with_retry('POST', url, payload)
async def _call_participant_confirm(self, participant_name: str, transaction_id: str, params: Dict) -> bool:
"""调用参与者Confirm方法"""
participant_info = self.participants.get(participant_name)
if not participant_info:
return False
endpoint = participant_info['endpoint']
url = f"{endpoint}/tcc/confirm"
payload = {
'transaction_id': transaction_id,
'params': params
}
return await self._http_call_with_retry('POST', url, payload)
async def _call_participant_cancel(self, participant_name: str, transaction_id: str, params: Dict) -> bool:
"""调用参与者Cancel方法"""
participant_info = self.participants.get(participant_name)
if not participant_info:
return False
endpoint = participant_info['endpoint']
url = f"{endpoint}/tcc/cancel"
payload = {
'transaction_id': transaction_id,
'params': params
}
return await self._http_call_with_retry('POST', url, payload)
async def _http_call_with_retry(self, method: str, url: str, payload: Dict) -> bool:
"""带重试的HTTP调用"""
timeout = aiohttp.ClientTimeout(total=10)
for attempt in range(self.retry_count):
try:
async with aiohttp.ClientSession(timeout=timeout) as session:
if method == 'POST':
async with session.post(url, json=payload) as response:
if response.status == 200:
result = await response.json()
return result.get('success', False)
else:
self.logger.warning(f"HTTP请求失败: {url}, 状态码: {response.status}")
return False
except Exception as e:
self.logger.warning(f"HTTP请求异常 (尝试 {attempt + 1}): {url}, {e}")
if attempt < self.retry_count - 1:
await asyncio.sleep(2 ** attempt) # 指数退避
continue
return False
def get_transaction_status(self, transaction_id: str) -> Optional[Dict]:
"""获取事务状态"""
transaction = self.transactions.get(transaction_id)
if not transaction:
return None
return {
'transaction_id': transaction.transaction_id,
'status': transaction.status.value,
'participants': transaction.participants,
'created_at': transaction.created_at,
'results': transaction.results
}
# 分布式参与者服务示例
class DistributedTCCTParticipantService:
"""分布式TCC参与者服务"""
def __init__(self, name: str, port: int):
self.name = name
self.port = port
self.tcc_service = None # 实际业务TCC服务
self.logger = logging.getLogger(f"DistributedTCC-{name}")
self.app = self._create_app()
def _create_app(self):
"""创建FastAPI应用"""
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
app = FastAPI(title=f"{self.name} TCC Service")
class TCCRequest(BaseModel):
transaction_id: str
params: Dict
@app.post("/tcc/try")
async def tcc_try(request: TCCRequest):
try:
if self.tcc_service:
success = self.tcc_service.try_operation(request.transaction_id, request.params)
return {"success": success}
else:
# 模拟成功
self.logger.info(f"模拟Try成功: {request.transaction_id}")
return {"success": True}
except Exception as e:
self.logger.error(f"Try操作异常: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/tcc/confirm")
async def tcc_confirm(request: TCCRequest):
try:
if self.tcc_service:
success = self.tcc_service.confirm_operation(request.transaction_id, request.params)
return {"success": success}
else:
# 模拟成功
self.logger.info(f"模拟Confirm成功: {request.transaction_id}")
return {"success": True}
except Exception as e:
self.logger.error(f"Confirm操作异常: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.post("/tcc/cancel")
async def tcc_cancel(request: TCCRequest):
try:
if self.tcc_service:
success = self.tcc_service.cancel_operation(request.transaction_id, request.params)
return {"success": success}
else:
# 模拟成功
self.logger.info(f"模拟Cancel成功: {request.transaction_id}")
return {"success": True}
except Exception as e:
self.logger.error(f"Cancel操作异常: {e}")
raise HTTPException(status_code=500, detail=str(e))
@app.get("/health")
async def health_check():
return {"status": "healthy"}
return app
async def start_service(self):
"""启动服务"""
import uvicorn
config = uvicorn.Config(app=self.app, host="0.0.0.0", port=self.port)
server = uvicorn.Server(config)
await server.serve()
# 使用示例
async def distributed_tcc_example():
"""分布式TCC模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建分布式TCC事务管理器
transaction_manager = DistributedTCCTransactionManager(timeout=30, retry_count=3)
# 注册分布式参与者
await transaction_manager.register_participant("PaymentService", "http://localhost:8001")
await transaction_manager.register_participant("OrderService", "http://localhost:8002")
await transaction_manager.register_participant("InventoryService", "http://localhost:8003")
# 执行分布式订单事务
participants = ["PaymentService", "OrderService", "InventoryService"]
operations = [
{
"user_id": "user123",
"amount": 999.99,
"payment_method": "credit_card"
},
{
"order_id": "order123",
"user_id": "user123",
"items": ["laptop", "mouse"]
},
{
"items": [
{"product_id": "laptop", "quantity": 1},
{"product_id": "mouse", "quantity": 1}
]
}
]
success = await transaction_manager.execute_transaction(participants, operations)
if success:
print("分布式订单事务执行成功")
else:
print("分布式订单事务执行失败")
# 获取事务状态
if transaction_manager.transactions:
last_transaction = list(transaction_manager.transactions.values())[-1]
status = transaction_manager.get_transaction_status(last_transaction.transaction_id)
print(f"事务状态: {status}")
---
c.监控和幂等性
---
# TCC监控和幂等性实现
import threading
from collections import defaultdict
typing import Dict, List, Optional, Set, Callable
from functools import wraps
class TCCIdempotencyManager:
"""TCC幂等性管理器"""
def __init__(self):
self.processed_operations = {} # 已处理操作
self.lock = threading.RLock()
self.logger = logging.getLogger("TCCIdempotency")
def is_operation_processed(self, transaction_id: str, participant_name: str, operation_type: str) -> bool:
"""检查操作是否已处理"""
key = f"{transaction_id}:{participant_name}:{operation_type}"
with self.lock:
return key in self.processed_operations
def mark_operation_processed(self, transaction_id: str, participant_name: str, operation_type: str, result: bool):
"""标记操作已处理"""
key = f"{transaction_id}:{participant_name}:{operation_type}"
with self.lock:
self.processed_operations[key] = {
'result': result,
'timestamp': time.time()
}
self.logger.info(f"标记操作已处理: {key}")
def get_operation_result(self, transaction_id: str, participant_name: str, operation_type: str) -> Optional[bool]:
"""获取操作结果"""
key = f"{transaction_id}:{participant_name}:{operation_type}"
with self.lock:
if key in self.processed_operations:
return self.processed_operations[key]['result']
return None
def cleanup_old_operations(self, max_age: int = 3600):
"""清理过期操作记录"""
current_time = time.time()
with self.lock:
expired_keys = []
for key, info in self.processed_operations.items():
if current_time - info['timestamp'] > max_age:
expired_keys.append(key)
for key in expired_keys:
del self.processed_operations[key]
if expired_keys:
self.logger.info(f"清理了 {len(expired_keys)} 个过期操作记录")
def idempotent_tcc_operation(idempotency_manager: TCCIdempotencyManager):
"""TCC操作幂等性装饰器"""
def decorator(operation_func: Callable) -> Callable:
@wraps(operation_func)
def wrapper(self, transaction_id: str, params: Dict, *args, **kwargs):
# 解析操作类型
operation_name = operation_func.__name__
if 'try' in operation_name.lower():
operation_type = 'try'
elif 'confirm' in operation_name.lower():
operation_type = 'confirm'
elif 'cancel' in operation_name.lower():
operation_type = 'cancel'
else:
operation_type = 'unknown'
# 检查幂等性
participant_name = self.name
if idempotency_manager.is_operation_processed(transaction_id, participant_name, operation_type):
self.logger.info(f"操作已处理，返回之前结果: {transaction_id}, {operation_type}")
return idempotency_manager.get_operation_result(transaction_id, participant_name, operation_type)
# 执行操作
try:
result = operation_func(self, transaction_id, params, *args, **kwargs)
# 记录操作结果
idempotency_manager.mark_operation_processed(
transaction_id, participant_name, operation_type, result
)
return result
except Exception as e:
self.logger.error(f"操作执行异常: {e}")
raise
return wrapper
return decorator
class TCCMetricsCollector:
"""TCC指标收集器"""
def __init__(self):
self.metrics = defaultdict(lambda: {
'total': 0,
'success': 0,
'failure': 0,
'timeout': 0,
'avg_duration': 0.0,
'max_duration': 0.0,
'min_duration': float('inf')
})
self.lock = threading.RLock()
self.logger = logging.getLogger("TCCMetrics")
def record_operation(self, operation_type: str, participant_name: str, success: bool, duration: float):
"""记录操作指标"""
key = f"{operation_type}:{participant_name}"
with self.lock:
metrics = self.metrics[key]
metrics['total'] += 1
metrics['avg_duration'] = (metrics['avg_duration'] * (metrics['total'] - 1) + duration) / metrics['total']
metrics['max_duration'] = max(metrics['max_duration'], duration)
metrics['min_duration'] = min(metrics['min_duration'], duration)
if success:
metrics['success'] += 1
else:
metrics['failure'] += 1
self.logger.info(f"记录指标: {key}, 成功: {success}, 耗时: {duration:.3f}s")
def record_timeout(self, operation_type: str, participant_name: str):
"""记录超时指标"""
key = f"{operation_type}:{participant_name}"
with self.lock:
self.metrics[key]['timeout'] += 1
self.logger.warning(f"记录超时: {key}")
def get_metrics(self) -> Dict:
"""获取所有指标"""
with self.lock:
result = {}
for key, metrics in self.metrics.items():
operation_type, participant_name = key.split(':', 1)
success_rate = metrics['success'] / max(1, metrics['total'])
result[key] = {
'operation_type': operation_type,
'participant': participant_name,
'total_operations': metrics['total'],
'success_count': metrics['success'],
'failure_count': metrics['failure'],
'timeout_count': metrics['timeout'],
'success_rate': success_rate,
'average_duration': metrics['avg_duration'],
'max_duration': metrics['max_duration'],
'min_duration': metrics['min_duration'] if metrics['min_duration'] != float('inf') else 0.0
}
return result
def reset_metrics(self):
"""重置指标"""
with self.lock:
self.metrics.clear()
self.logger.info("指标已重置")
def tcc_metrics_collector(metrics_collector: TCCMetricsCollector):
"""TCC指标收集装饰器"""
def decorator(operation_func: Callable) -> Callable:
@wraps(operation_func)
def wrapper(self, transaction_id: str, params: Dict, *args, **kwargs):
# 解析操作类型
operation_name = operation_func.__name__
if 'try' in operation_name.lower():
operation_type = 'try'
elif 'confirm' in operation_name.lower():
operation_type = 'confirm'
elif 'cancel' in operation_name.lower():
operation_type = 'cancel'
else:
operation_type = 'unknown'
participant_name = self.name
start_time = time.time()
try:
result = operation_func(self, transaction_id, params, *args, **kwargs)
duration = time.time() - start_time
metrics_collector.record_operation(operation_type, participant_name, result, duration)
return result
except Exception as e:
duration = time.time() - start_time
metrics_collector.record_operation(operation_type, participant_name, False, duration)
raise
return wrapper
return decorator
class MonitoredBankingTCCService(BankingTCCService):
"""带监控的银行TCC服务"""
def __init__(self, name: str, accounts: Dict[str, float], idempotency_manager: TCCIdempotencyManager, metrics_collector: TCCMetricsCollector):
super().__init__(name, accounts)
self.idempotency_manager = idempotency_manager
self.metrics_collector = metrics_collector
@idempotent_tcc_operation(idempotency_manager)
@tcc_metrics_collector(metrics_collector)
def try_operation(self, transaction_id: str, params: Dict) -> bool:
return super().try_operation(transaction_id, params)
@idempotent_tcc_operation(idempotency_manager)
@tcc_metrics_collector(metrics_collector)
def confirm_operation(self, transaction_id: str, params: Dict) -> bool:
return super().confirm_operation(transaction_id, params)
@idempotent_tcc_operation(idempotency_manager)
@tcc_metrics_collector(metrics_collector)
def cancel_operation(self, transaction_id: str, params: Dict) -> bool:
return super().cancel_operation(transaction_id, params)
class AdvancedTCCTransactionManager(TCCTransactionManager):
"""高级TCC事务管理器，支持监控和幂等性"""
def __init__(self, timeout: int = 30):
super().__init__(timeout)
self.idempotency_manager = TCCIdempotencyManager()
self.metrics_collector = TCCMetricsCollector()
self.transaction_monitor = TCCTransactionMonitor()
def register_monitored_participant(self, participant):
"""注册带监控的参与者"""
if isinstance(participant, MonitoredBankingTCCService):
self.register_participant(participant)
else:
self.logger.warning(f"参与者类型不支持监控: {type(participant)}")
def get_performance_report(self) -> Dict:
"""获取性能报告"""
metrics = self.metrics_collector.get_metrics()
# 按操作类型分组统计
try_stats = defaultdict(lambda: {'total': 0, 'success': 0, 'failure': 0, 'avg_duration': 0.0})
confirm_stats = defaultdict(lambda: {'total': 0, 'success': 0, 'failure': 0, 'avg_duration': 0.0})
cancel_stats = defaultdict(lambda: {'total': 0, 'success': 0, 'failure': 0, 'avg_duration': 0.0})
for key, metric in metrics.items():
if metric['operation_type'] == 'try':
stats = try_stats['overall']
elif metric['operation_type'] == 'confirm':
stats = confirm_stats['overall']
elif metric['operation_type'] == 'cancel':
stats = cancel_stats['overall']
else:
continue
stats['total'] += metric['total_operations']
stats['success'] += metric['success_count']
stats['failure'] += metric['failure_count']
stats['avg_duration'] += metric['average_duration'] * metric['total_operations']
# 计算平均值
for stats in [try_stats['overall'], confirm_stats['overall'], cancel_stats['overall']]:
if stats['total'] > 0:
stats['avg_duration'] /= stats['total']
return {
'try_operations': dict(try_stats),
'confirm_operations': dict(confirm_stats),
'cancel_operations': dict(cancel_stats),
'detailed_metrics': metrics
}
class TCCTransactionMonitor:
"""TCC事务监控器"""
def __init__(self):
self.active_transactions = {}
self.completed_transactions = []
self.lock = threading.RLock()
self.logger = logging.getLogger("TCCMonitor")
def start_transaction(self, transaction_id: str, participants: List[str]):
"""开始监控事务"""
with self.lock:
self.active_transactions[transaction_id] = {
'participants': participants,
'start_time': time.time(),
'status': TCCStatus.TRYING
}
self.logger.info(f"开始监控事务: {transaction_id}")
def update_transaction_status(self, transaction_id: str, status: TCCStatus):
"""更新事务状态"""
with self.lock:
if transaction_id in self.active_transactions:
self.active_transactions[transaction_id]['status'] = status
if status in [TCCStatus.CONFIRMED, TCCStatus.CANCELLED]:
# 事务完成，移到历史记录
transaction = self.active_transactions.pop(transaction_id)
transaction['end_time'] = time.time()
transaction['duration'] = transaction['end_time'] - transaction['start_time']
self.completed_transactions.append(transaction)
self.logger.info(f"事务完成: {transaction_id}, 状态: {status.value}, 耗时: {transaction['duration']:.3f}s")
def get_active_transactions(self) -> List[Dict]:
"""获取活跃事务"""
with self.lock:
return [
{
'transaction_id': tx_id,
'participants': info['participants'],
'status': info['status'].value,
'duration': time.time() - info['start_time']
}
for tx_id, info in self.active_transactions.items()
]
def get_transaction_summary(self) -> Dict:
"""获取事务摘要统计"""
with self.lock:
total_transactions = len(self.completed_transactions)
if total_transactions == 0:
return {
'total_transactions': 0,
'average_duration': 0.0,
'success_rate': 0.0
}
confirmed_count = sum(1 for tx in self.completed_transactions if tx['status'] == TCCStatus.CONFIRMED)
total_duration = sum(tx['duration'] for tx in self.completed_transactions)
return {
'total_transactions': total_transactions,
'confirmed_transactions': confirmed_count,
'cancelled_transactions': total_transactions - confirmed_count,
'success_rate': confirmed_count / total_transactions,
'average_duration': total_duration / total_transactions,
'active_transactions': len(self.active_transactions)
}
# 使用示例
def monitored_tcc_example():
"""带监控的TCC模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建监控组件
idempotency_manager = TCCIdempotencyManager()
metrics_collector = TCCMetricsCollector()
# 创建带监控的服务
banking_service = MonitoredBankingTCCService(
"MonitoredBank",
{"user001": 15000.0, "merchant001": 8000.0},
idempotency_manager,
metrics_collector
)
# 创建高级事务管理器
transaction_manager = AdvancedTCCTransactionManager(timeout=30)
transaction_manager.register_monitored_participant(banking_service)
# 执行多个事务
print("开始执行多个带监控的TCC事务...")
for i in range(3):
participants = ["MonitoredBank"]
operations = [{
"account_id": "user001",
"amount": 1000.0 + i * 500,
"operation_type": "debit"
}]
success = transaction_manager.execute_transaction(participants, operations)
print(f"事务 {i+1}: {'成功' if success else '失败'}")
# 重复执行相同事务测试幂等性
print("\n测试幂等性（重复执行相同事务）...")
participants = ["MonitoredBank"]
operations = [{
"account_id": "user001",
"amount": 500.0,
"operation_type": "debit"
}]
# 第一次执行
success1 = transaction_manager.execute_transaction(participants, operations)
print(f"第一次执行: {'成功' if success1 else '失败'}")
# 第二次执行相同事务（应该被幂等性处理）
success2 = transaction_manager.execute_transaction(participants, operations)
print(f"第二次执行（幂等性测试）: {'成功' if success2 else '失败'}")
# 获取性能报告
print("\n性能报告:")
report = transaction_manager.get_performance_report()
print(f"Try操作统计: {report['try_operations']}")
print(f"Confirm操作统计: {report['confirm_operations']}")
print(f"Cancel操作统计: {report['cancel_operations']}")
# 获取详细指标
print("\n详细指标:")
for key, metric in report['detailed_metrics'].items():
print(f"{key}: 成功率 {metric['success_rate']:.2%}, 平均耗时 {metric['average_duration']:.3f}s")
print(f"\n用户余额: {banking_service.get_balance('user001')}")
# 清理过期操作记录
idempotency_manager.cleanup_old_operations()
---
06.性能优化
a.并发控制
使用异步并发处理TCC操作，减少事务执行时间。合理控制并发度，避免过度并发导致系统资源竞争。
b.资源优化
优化Try阶段的资源预留策略，减少资源锁定时间。使用更轻量级的预留机制，如内存缓存或分布式锁。
c.监控告警
建立完善的TCC事务监控体系，实时跟踪事务执行状态。设置合理的告警阈值，及时发现和处理异常情况。
6.4 SAGA模式
01.SAGA模式概述
a.基本概念
SAGA模式是一种长事务分解模式，将分布式事务拆分为一系列本地事务，每个本地事务都有对应的补偿操作。通过补偿机制保证最终一致性，适用于业务流程较长的场景。
b.模式原理
SAGA通过将全局事务分解为多个子事务，每个子事务在各自的数据源中执行。如果某个子事务失败，则按照相反顺序执行之前已成功子事务的补偿操作，实现回滚效果。
c.核心特性
支持长时间运行的分布式事务，避免资源长时间锁定。通过补偿操作实现最终一致性，不依赖数据库的事务机制，提高了系统的可用性和扩展性。
02.补偿机制
a.补偿事务设计
每个业务操作必须设计对应的补偿操作，能够将系统状态恢复到操作执行前的状态。补偿操作需要保证幂等性，确保重复执行不会产生副作用。
b.补偿执行策略
当业务操作失败时，按照与执行相反的顺序执行补偿操作。补偿操作也需要处理各种异常情况，确保系统能够安全回滚到一致状态。
c.补偿原子性
补偿操作本身需要保证原子性，要么完全成功，要么完全失败。通过重试机制和状态检查确保补偿操作的可靠性。
03.SAGA编排
a.中央编排模式
使用中央协调器管理整个SAGA流程，控制各个子事务的执行顺序和补偿操作。协调器负责状态管理、异常处理和补偿触发。
b.协同模式
各个服务通过消息传递协同完成SAGA流程，每个服务执行自己的事务并通知其他服务。不依赖中央协调器，提高了系统的去中心化程度。
c.混合模式
结合中央编排和协同模式的优势，在复杂业务流程中使用编排器管理主要流程，同时允许服务间直接通信提高效率。
04.状态管理
a.事务状态跟踪
跟踪每个子事务的执行状态，包括待执行、执行中、已完成、补偿中、已补偿等。通过状态机管理事务生命周期。
b.持久化存储
将事务状态持久化到数据库或其他存储系统，确保系统崩溃后能够恢复事务状态并继续执行。
c.状态恢复
系统重启后能够从存储中恢复未完成的事务状态，继续执行或补偿操作，保证事务的最终完成。
05.实现示例
a.基础SAGA编排器
---
# 基础SAGA模式实现
import logging
import time
import uuid
from typing import Dict, List, Optional, Callable, Any
from enum import Enum
from dataclasses import dataclass
from abc import ABC, abstractmethod
class SagaStepStatus(Enum):
"""SAGA步骤状态"""
PENDING = "PENDING"
RUNNING = "RUNNING"
COMPLETED = "COMPLETED"
COMPENSATING = "COMPENSATING"
COMPENSATED = "COMPENSATED"
FAILED = "FAILED"
class SagaStatus(Enum):
"""SAGA状态"""
RUNNING = "RUNNING"
COMPLETED = "COMPLETED"
COMPENSATING = "COMPENSATING"
COMPENSATED = "COMPENSATED"
FAILED = "FAILED"
@dataclass
class SagaStep:
"""SAGA步骤"""
step_id: str
name: str
execute_func: Callable
compensate_func: Callable
params: Dict
status: SagaStepStatus = SagaStepStatus.PENDING
result: Any = None
error: Optional[str] = None
execute_time: Optional[float] = None
compensate_time: Optional[float] = None
@dataclass
class Saga:
"""SAGA事务"""
saga_id: str
name: str
steps: List[SagaStep]
current_step_index: int = -1
status: SagaStatus = SagaStatus.RUNNING
created_at: float = 0
updated_at: float = 0
error: Optional[str] = None
class SagaOrchestrator:
"""SAGA编排器"""
def __init__(self):
self.sagas = {}
self.logger = logging.getLogger("SagaOrchestrator")
def create_saga(self, name: str) -> str:
"""创建SAGA"""
saga_id = str(uuid.uuid4())
saga = Saga(
saga_id=saga_id,
name=name,
steps=[],
created_at=time.time(),
updated_at=time.time()
)
self.sagas[saga_id] = saga
self.logger.info(f"创建SAGA: {saga_id} - {name}")
return saga_id
def add_step(self, saga_id: str, name: str, execute_func: Callable, compensate_func: Callable, params: Dict) -> str:
"""添加SAGA步骤"""
if saga_id not in self.sagas:
raise ValueError(f"SAGA不存在: {saga_id}")
saga = self.sagas[saga_id]
step_id = str(uuid.uuid4())
step = SagaStep(
step_id=step_id,
name=name,
execute_func=execute_func,
compensate_func=compens_func,
params=params
)
saga.steps.append(step)
saga.updated_at = time.time()
self.logger.info(f"添加步骤: {step_id} - {name}")
return step_id
def execute_saga(self, saga_id: str) -> bool:
"""执行SAGA"""
if saga_id not in self.sagas:
self.logger.error(f"SAGA不存在: {saga_id}")
return False
saga = self.sagas[saga_id]
self.logger.info(f"开始执行SAGA: {saga_id} - {saga.name}")
try:
saga.status = SagaStatus.RUNNING
saga.updated_at = time.time()
# 执行所有步骤
for i, step in enumerate(saga.steps):
saga.current_step_index = i
if not self._execute_step(saga, step):
# 步骤失败，开始补偿
self.logger.error(f"步骤失败，开始补偿: {step.name}")
self._compensate_saga(saga)
return False
# 所有步骤成功
saga.status = SagaStatus.COMPLETED
saga.updated_at = time.time()
self.logger.info(f"SAGA执行成功: {saga_id}")
return True
except Exception as e:
self.logger.error(f"SAGA执行异常: {saga_id}, {e}")
saga.error = str(e)
saga.status = SagaStatus.FAILED
saga.updated_at = time.time()
return False
def _execute_step(self, saga: Saga, step: SagaStep) -> bool:
"""执行单个步骤"""
self.logger.info(f"执行步骤: {step.name}")
step.status = SagaStepStatus.RUNNING
step.execute_time = time.time()
saga.updated_at = time.time()
try:
# 执行业务逻辑
result = step.execute_func(step.params)
step.result = result
step.status = SagaStepStatus.COMPLETED
step.execute_time = time.time() - step.execute_time
saga.updated_at = time.time()
self.logger.info(f"步骤执行成功: {step.name}, 耗时: {step.execute_time:.3f}s")
return True
except Exception as e:
step.status = SagaStepStatus.FAILED
step.error = str(e)
step.execute_time = time.time() - step.execute_time
saga.updated_at = time.time()
self.logger.error(f"步骤执行失败: {step.name}, 错误: {e}")
return False
def _compensate_saga(self, saga: Saga):
"""补偿SAGA"""
self.logger.info(f"开始补偿SAGA: {saga.saga_id}")
saga.status = SagaStatus.COMPENSATING
saga.updated_at = time.time()
# 按相反顺序补偿已完成的步骤
completed_steps = [step for step in saga.steps if step.status == SagaStepStatus.COMPLETED]
for step in reversed(completed_steps):
if not self._compensate_step(saga, step):
self.logger.error(f"补偿步骤失败: {step.name}")
# 补偿失败需要人工介入
saga.error = f"补偿步骤失败: {step.name}"
saga.status = SagaStatus.FAILED
saga.updated_at = time.time()
return
# 所有补偿完成
saga.status = SagaStatus.COMPENSATED
saga.updated_at = time.time()
self.logger.info(f"SAGA补偿完成: {saga.saga_id}")
def _compensate_step(self, saga: Saga, step: SagaStep) -> bool:
"""补偿单个步骤"""
self.logger.info(f"补偿步骤: {step.name}")
step.status = SagaStepStatus.COMPENSATING
step.compensate_time = time.time()
try:
# 执行补偿逻辑
result = step.compensate_func(step.params, step.result)
step.status = SagaStepStatus.COMPENSATED
step.compensate_time = time.time() - step.compensate_time
saga.updated_at = time.time()
self.logger.info(f"步骤补偿成功: {step.name}, 耗时: {step.compensate_time:.3f}s")
return True
except Exception as e:
step.error = str(e)
step.compensate_time = time.time() - step.compensate_time
saga.updated_at = time.time()
self.logger.error(f"步骤补偿失败: {step.name}, 错误: {e}")
return False
def get_saga_status(self, saga_id: str) -> Optional[Dict]:
"""获取SAGA状态"""
if saga_id not in self.sagas:
return None
saga = self.sagas[saga_id]
return {
'saga_id': saga.saga_id,
'name': saga.name,
'status': saga.status.value,
'current_step_index': saga.current_step_index,
'total_steps': len(saga.steps),
'completed_steps': len([s for s in saga.steps if s.status == SagaStepStatus.COMPLETED]),
'failed_steps': len([s for s in saga.steps if s.status == SagaStepStatus.FAILED]),
'compensated_steps': len([s for s in saga.steps if s.status == SagaStepStatus.COMPENSATED]),
'created_at': saga.created_at,
'updated_at': saga.updated_at,
'error': saga.error,
'steps': [
{
'step_id': step.step_id,
'name': step.name,
'status': step.status.value,
'execute_time': step.execute_time,
'compensate_time': step.compensate_time,
'error': step.error
}
for step in saga.steps
]
}
# 业务服务示例
class OrderService:
"""订单服务"""
def __init__(self):
self.orders = {}
self.logger = logging.getLogger("OrderService")
def create_order(self, params: Dict) -> str:
"""创建订单"""
order_id = f"ORD_{uuid.uuid4().hex[:8]}"
order = {
'order_id': order_id,
'user_id': params['user_id'],
'items': params['items'],
'total_amount': params['total_amount'],
'status': 'PENDING',
'created_at': time.time()
}
self.orders[order_id] = order
self.logger.info(f"创建订单: {order_id}")
return order_id
def cancel_order(self, params: Dict, result: Any) -> bool:
"""取消订单"""
order_id = result
if order_id in self.orders:
self.orders[order_id]['status'] = 'CANCELLED'
self.logger.info(f"取消订单: {order_id}")
return True
return False
class PaymentService:
"""支付服务"""
def __init__(self):
self.payments = {}
self.accounts = {
'user001': 10000.0,
'merchant001': 0.0
}
self.logger = logging.getLogger("PaymentService")
def process_payment(self, params: Dict) -> str:
"""处理支付"""
user_id = params['user_id']
merchant_id = params['merchant_id']
amount = params['amount']
if self.accounts.get(user_id, 0) < amount:
raise ValueError(f"余额不足: {self.accounts.get(user_id, 0)}")
# 扣款
self.accounts[user_id] -= amount
self.accounts[merchant_id] += amount
payment_id = f"PAY_{uuid.uuid4().hex[:8]}"
payment = {
'payment_id': payment_id,
'user_id': user_id,
'merchant_id': merchant_id,
'amount': amount,
'status': 'COMPLETED',
'created_at': time.time()
}
self.payments[payment_id] = payment
self.logger.info(f"处理支付: {payment_id}, 金额: {amount}")
return payment_id
def refund_payment(self, params: Dict, result: Any) -> bool:
"""退款"""
payment_id = result
if payment_id not in self.payments:
return False
payment = self.payments[payment_id]
user_id = payment['user_id']
merchant_id = payment['merchant_id']
amount = payment['amount']
# 退款
self.accounts[user_id] += amount
self.accounts[merchant_id] -= amount
payment['status'] = 'REFUNDED'
self.logger.info(f"退款支付: {payment_id}, 金额: {amount}")
return True
class InventoryService:
"""库存服务"""
def __init__(self):
self.inventory = {
'laptop': 100,
'mouse': 200,
'keyboard': 150
}
self.reservations = {}
self.logger = logging.getLogger("InventoryService")
def reserve_inventory(self, params: Dict) -> str:
"""预留库存"""
reservation_id = f"INV_{uuid.uuid4().hex[:8]}"
items = params['items']
# 检查库存
for item in items:
product_id = item['product_id']
quantity = item['quantity']
if self.inventory.get(product_id, 0) < quantity:
raise ValueError(f"库存不足: {product_id}")
# 预留库存
for item in items:
product_id = item['product_id']
quantity = item['quantity']
self.inventory[product_id] -= quantity
reservation = {
'reservation_id': reservation_id,
'items': items,
'status': 'RESERVED',
'created_at': time.time()
}
self.reservations[reservation_id] = reservation
self.logger.info(f"预留库存: {reservation_id}")
return reservation_id
def release_inventory(self, params: Dict, result: Any) -> bool:
"""释放库存"""
reservation_id = result
if reservation_id not in self.reservations:
return False
reservation = self.reservations[reservation_id]
items = reservation['items']
# 释放库存
for item in items:
product_id = item['product_id']
quantity = item['quantity']
self.inventory[product_id] += quantity
reservation['status'] = 'RELEASED'
self.logger.info(f"释放库存: {reservation_id}")
return True
# 使用示例
def basic_saga_example():
"""基础SAGA模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建服务实例
order_service = OrderService()
payment_service = PaymentService()
inventory_service = InventoryService()
# 创建编排器
orchestrator = SagaOrchestrator()
# 创建订单处理SAGA
saga_id = orchestrator.create_saga("订单处理流程")
# 添加步骤：创建订单
orchestrator.add_step(
saga_id=saga_id,
name="创建订单",
execute_func=order_service.create_order,
compensate_func=order_service.cancel_order,
params={
'user_id': 'user001',
'items': ['laptop', 'mouse'],
'total_amount': 5999.99
}
)
# 添加步骤：处理支付
orchestrator.add_step(
saga_id=saga_id,
name="处理支付",
execute_func=payment_service.process_payment,
compensate_func=payment_service.refund_payment,
params={
'user_id': 'user001',
'merchant_id': 'merchant001',
'amount': 5999.99
}
)
# 添加步骤：预留库存
orchestrator.add_step(
saga_id=saga_id,
name="预留库存",
execute_func=inventory_service.reserve_inventory,
compensate_func=inventory_service.release_inventory,
params={
'items': [
{'product_id': 'laptop', 'quantity': 1},
{'product_id': 'mouse', 'quantity': 1}
]
}
)
print(f"执行前状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"库存 - 笔记本: {inventory_service.inventory['laptop']}, 鼠标: {inventory_service.inventory['mouse']}")
print(f"订单数量: {len(order_service.orders)}")
# 执行SAGA
success = orchestrator.execute_saga(saga_id)
# 获取状态
status = orchestrator.get_saga_status(saga_id)
print(f"\n执行结果: {'成功' if success else '失败'}")
print(f"SAGA状态: {status['status']}")
print(f"已完成步骤: {status['completed_steps']}/{status['total_steps']}")
print(f"\n执行后状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"库存 - 笔记本: {inventory_service.inventory['laptop']}, 鼠标: {inventory_service.inventory['mouse']}")
print(f"订单数量: {len(order_service.orders)}")
if success:
print("订单处理成功!")
else:
print("订单处理失败，已回滚")
---
b.协同式SAGA
---
# 协同式SAGA实现
import asyncio
import json
from typing import Dict, List, Optional, Callable
from dataclasses import dataclass, asdict
from enum import Enum
class SagaEventType(Enum):
"""SAGA事件类型"""
STARTED = "STARTED"
STEP_STARTED = "STEP_STARTED"
STEP_COMPLETED = "STEP_COMPLETED"
STEP_FAILED = "STEP_FAILED"
COMPENSATION_STARTED = "COMPENSATION_STARTED"
COMPENSATION_COMPLETED = "COMPENSATION_COMPLETED"
COMPENSATION_FAILED = "COMPENSATION_FAILED"
SAGA_COMPLETED = "SAGA_COMPLETED"
SAGA_FAILED = "SAGA_FAILED"
@dataclass
class SagaEvent:
"""SAGA事件"""
event_id: str
saga_id: str
step_name: Optional[str]
event_type: SagaEventType
timestamp: float
data: Dict
class SagaEventBus:
"""SAGA事件总线"""
def __init__(self):
self.subscribers = {}
self.events = []
self.logger = logging.getLogger("SagaEventBus")
def subscribe(self, event_type: SagaEventType, handler: Callable):
"""订阅事件"""
if event_type not in self.subscribers:
self.subscribers[event_type] = []
self.subscribers[event_type].append(handler)
self.logger.info(f"订阅事件: {event_type.value}")
def publish(self, event: SagaEvent):
"""发布事件"""
self.events.append(event)
self.logger.info(f"发布事件: {event.event_type.value} - {event.saga_id}")
# 通知订阅者
handlers = self.subscribers.get(event.event_type, [])
for handler in handlers:
try:
asyncio.create_task(handler(event))
except Exception as e:
self.logger.error(f"事件处理异常: {e}")
class ChoreographySagaParticipant:
"""协同式SAGA参与者"""
def __init__(self, name: str, event_bus: SagaEventBus):
self.name = name
self.event_bus = event_bus
self.active_sagas = {}
self.logger = logging.getLogger(f"ChoreographySaga-{name}")
def start_saga(self, saga_id: str, initial_data: Dict):
"""开始SAGA"""
self.active_sagas[saga_id] = {
'status': 'running',
'steps': [],
'data': initial_data.copy()
}
# 发布SAGA开始事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=self.name,
event_type=SagaEventType.STARTED,
timestamp=time.time(),
data=initial_data
)
self.event_bus.publish(event)
self.logger.info(f"开始SAGA: {saga_id}")
def execute_step(self, saga_id: str, step_name: str, execute_func: Callable, compensate_func: Callable, params: Dict):
"""执行步骤"""
if saga_id not in self.active_sagas:
self.logger.error(f"SAGA不存在: {saga_id}")
return False
# 发布步骤开始事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=step_name,
event_type=SagaEventType.STEP_STARTED,
timestamp=time.time(),
data=params
)
self.event_bus.publish(event)
try:
# 执行业务逻辑
result = execute_func(params)
# 记录步骤信息
step_info = {
'step_name': step_name,
'execute_func': execute_func,
'compensate_func': compensate_func,
'params': params,
'result': result,
'status': 'completed'
}
self.active_sagas[saga_id]['steps'].append(step_info)
# 发布步骤完成事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=step_name,
event_type=SagaEventType.STEP_COMPLETED,
timestamp=time.time(),
data={'result': result}
)
self.event_bus.publish(event)
self.logger.info(f"步骤完成: {step_name}")
return True
except Exception as e:
# 记录失败步骤
step_info = {
'step_name': step_name,
'execute_func': execute_func,
'compensate_func': compensate_func,
'params': params,
'result': None,
'status': 'failed',
'error': str(e)
}
self.active_sagas[saga_id]['steps'].append(step_info)
# 发布步骤失败事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=step_name,
event_type=SagaEventType.STEP_FAILED,
timestamp=time.time(),
data={'error': str(e)}
)
self.event_bus.publish(event)
self.logger.error(f"步骤失败: {step_name}, 错误: {e}")
return False
async def compensate_saga(self, saga_id: str):
"""补偿SAGA"""
if saga_id not in self.active_sagas:
return
saga = self.active_sagas[saga_id]
completed_steps = [step for step in saga['steps'] if step['status'] == 'completed']
# 发布补偿开始事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=None,
event_type=SagaEventType.COMPENSATION_STARTED,
timestamp=time.time(),
data={}
)
self.event_bus.publish(event)
# 按相反顺序补偿
for step in reversed(completed_steps):
try:
compensate_func = step['compensate_func']
params = step['params']
result = step['result']
# 执行补偿
compensate_result = compensate_func(params, result)
self.logger.info(f"补偿步骤完成: {step['step_name']}")
except Exception as e:
# 发布补偿失败事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=step['step_name'],
event_type=SagaEventType.COMPENSATION_FAILED,
timestamp=time.time(),
data={'error': str(e)}
)
self.event_bus.publish(event)
self.logger.error(f"补偿步骤失败: {step['step_name']}, 错误: {e}")
return False
saga['status'] = 'compensated'
# 发布补偿完成事件
event = SagaEvent(
event_id=str(uuid.uuid4()),
saga_id=saga_id,
step_name=None,
event_type=SagaEventType.COMPENSATION_COMPLETED,
timestamp=time.time(),
data={}
)
self.event_bus.publish(event)
self.logger.info(f"SAGA补偿完成: {saga_id}")
class OrderServiceParticipant(ChoreographySagaParticipant):
"""订单服务参与者"""
def __init__(self, event_bus: SagaEventBus):
super().__init__("OrderService", event_bus)
self.orders = {}
self._setup_event_handlers()
def _setup_event_handlers(self):
"""设置事件处理器"""
self.event_bus.subscribe(SagaEventType.STARTED, self._handle_saga_started)
self.event_bus.subscribe(SagaEventType.COMPENSATION_STARTED, self._handle_compensation_started)
async def _handle_saga_started(self, event: SagaEvent):
"""处理SAGA开始事件"""
if event.data.get('service') == 'OrderService':
await self._create_order(event.saga_id, event.data)
async def _handle_compensation_started(self, event: SagaEvent):
"""处理补偿开始事件"""
await self._cancel_orders(event.saga_id)
async def _create_order(self, saga_id: str, data: Dict):
"""创建订单"""
def create_order_func(params: Dict) -> str:
order_id = f"ORD_{uuid.uuid4().hex[:8]}"
order = {
'order_id': order_id,
'saga_id': saga_id,
'user_id': params['user_id'],
'items': params['items'],
'total_amount': params['total_amount'],
'status': 'PENDING',
'created_at': time.time()
}
self.orders[order_id] = order
self.logger.info(f"创建订单: {order_id}")
return order_id
def cancel_order_func(params: Dict, result: Any) -> bool:
order_id = result
if order_id in self.orders:
self.orders[order_id]['status'] = 'CANCELLED'
self.logger.info(f"取消订单: {order_id}")
return True
return False
self.execute_step(
saga_id=saga_id,
step_name="创建订单",
execute_func=create_order_func,
compensate_func=cancel_order_func,
params=data
)
async def _cancel_orders(self, saga_id: str):
"""取消订单"""
for order_id, order in self.orders.items():
if order.get('saga_id') == saga_id and order['status'] == 'PENDING':
order['status'] = 'CANCELLED'
self.logger.info(f"取消订单: {order_id}")
class PaymentServiceParticipant(ChoreographySagaParticipant):
"""支付服务参与者"""
def __init__(self, event_bus: SagaEventBus):
super().__init__("PaymentService", event_bus)
self.payments = {}
self.accounts = {
'user001': 10000.0,
'merchant001': 0.0
}
self._setup_event_handlers()
def _setup_event_handlers(self):
"""设置事件处理器"""
self.event_bus.subscribe(SagaEventType.STEP_COMPLETED, self._handle_step_completed)
self.event_bus.subscribe(SagaEventType.COMPENSATION_STARTED, self._handle_compensation_started)
async def _handle_step_completed(self, event: SagaEvent):
"""处理步骤完成事件"""
if event.step_name == "创建订单":
# 订单创建完成后开始支付
await self._process_payment(event.saga_id, event.data['result'])
async def _handle_compensation_started(self, event: SagaEvent):
"""处理补偿开始事件"""
await self._refund_payments(event.saga_id)
async def _process_payment(self, saga_id: str, order_id: str):
"""处理支付"""
def process_payment_func(params: Dict) -> str:
user_id = params['user_id']
merchant_id = params['merchant_id']
amount = params['amount']
if self.accounts.get(user_id, 0) < amount:
raise ValueError(f"余额不足: {self.accounts.get(user_id, 0)}")
# 执行支付
self.accounts[user_id] -= amount
self.accounts[merchant_id] += amount
payment_id = f"PAY_{uuid.uuid4().hex[:8]}"
payment = {
'payment_id': payment_id,
'saga_id': saga_id,
'order_id': order_id,
'user_id': user_id,
'merchant_id': merchant_id,
'amount': amount,
'status': 'COMPLETED',
'created_at': time.time()
}
self.payments[payment_id] = payment
self.logger.info(f"处理支付: {payment_id}, 金额: {amount}")
return payment_id
def refund_payment_func(params: Dict, result: Any) -> bool:
payment_id = result
if payment_id not in self.payments:
return False
payment = self.payments[payment_id]
user_id = payment['user_id']
merchant_id = payment['merchant_id']
amount = payment['amount']
# 执行退款
self.accounts[user_id] += amount
self.accounts[merchant_id] -= amount
payment['status'] = 'REFUNDED'
self.logger.info(f"退款支付: {payment_id}, 金额: {amount}")
return True
self.execute_step(
saga_id=saga_id,
step_name="处理支付",
execute_func=process_payment_func,
compensate_func=refund_payment_func,
params={
'user_id': 'user001',
'merchant_id': 'merchant001',
'amount': 5999.99,
'order_id': order_id
}
)
async def _refund_payments(self, saga_id: str):
"""退款支付"""
for payment_id, payment in self.payments.items():
if payment.get('saga_id') == saga_id and payment['status'] == 'COMPLETED':
user_id = payment['user_id']
merchant_id = payment['merchant_id']
amount = payment['amount']
# 执行退款
self.accounts[user_id] += amount
self.accounts[merchant_id] -= amount
payment['status'] = 'REFUNDED'
self.logger.info(f"退款支付: {payment_id}")
# 使用示例
async def choreography_saga_example():
"""协同式SAGA模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建事件总线
event_bus = SagaEventBus()
# 创建参与者
order_service = OrderServiceParticipant(event_bus)
payment_service = PaymentServiceParticipant(event_bus)
# 启动SAGA
saga_id = str(uuid.uuid4())
order_data = {
'service': 'OrderService',
'user_id': 'user001',
'items': ['laptop', 'mouse'],
'total_amount': 5999.99
}
print(f"执行前状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"订单数量: {len(order_service.orders)}")
# 开始SAGA
order_service.start_saga(saga_id, order_data)
# 等待SAGA执行
await asyncio.sleep(2)
# 检查最终状态
print(f"\n执行后状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"订单数量: {len(order_service.orders)}")
print(f"支付数量: {len(payment_service.payments)}")
# 显示事件历史
print(f"\n事件历史:")
for event in event_bus.events[-10:]: # 显示最后10个事件
print(f"{event.event_type.value}: {event.saga_id} - {event.step_name or 'N/A'}")
---
c.持久化和恢复
---
# SAGA持久化和恢复实现
import pickle
import os
from pathlib import Path
typing import Dict, List, Optional, Any
from datetime import datetime, timedelta
class SagaPersistence:
"""SAGA持久化管理"""
def __init__(self, storage_path: str = "./saga_storage"):
self.storage_path = Path(storage_path)
self.storage_path.mkdir(exist_ok=True)
self.logger = logging.getLogger("SagaPersistence")
def save_saga(self, saga: Saga):
"""保存SAGA状态"""
saga_file = self.storage_path / f"{saga.saga_id}.pkl"
try:
with open(saga_file, 'wb') as f:
pickle.dump(saga, f)
self.logger.info(f"保存SAGA状态: {saga.saga_id}")
except Exception as e:
self.logger.error(f"保存SAGA状态失败: {e}")
def load_saga(self, saga_id: str) -> Optional[Saga]:
"""加载SAGA状态"""
saga_file = self.storage_path / f"{saga_id}.pkl"
if not saga_file.exists():
return None
try:
with open(saga_file, 'rb') as f:
saga = pickle.load(f)
self.logger.info(f"加载SAGA状态: {saga_id}")
return saga
except Exception as e:
self.logger.error(f"加载SAGA状态失败: {e}")
return None
def delete_saga(self, saga_id: str):
"""删除SAGA状态"""
saga_file = self.storage_path / f"{saga_id}.pkl"
try:
if saga_file.exists():
saga_file.unlink()
self.logger.info(f"删除SAGA状态: {saga_id}")
except Exception as e:
self.logger.error(f"删除SAGA状态失败: {e}")
def get_all_sagas(self) -> List[str]:
"""获取所有SAGA ID"""
try:
saga_files = list(self.storage_path.glob("*.pkl"))
return [f.stem for f in saga_files]
except Exception as e:
self.logger.error(f"获取SAGA列表失败: {e}")
return []
def get_unfinished_sagas(self, max_age_hours: int = 24) -> List[str]:
"""获取未完成的SAGA"""
unfinished = []
current_time = time.time()
for saga_id in self.get_all_sagas():
saga = self.load_saga(saga_id)
if saga and saga.status in [SagaStatus.RUNNING, SagaStatus.COMPENSATING]:
# 检查年龄
age_hours = (current_time - saga.created_at) / 3600
if age_hours < max_age_hours:
unfinished.append(saga_id)
return unfinished
class PersistentSagaOrchestrator(SagaOrchestrator):
"""持久化SAGA编排器"""
def __init__(self, persistence_path: str = "./saga_storage"):
super().__init__()
self.persistence = SagaPersistence(persistence_path)
self._load_existing_sagas()
def _load_existing_sagas(self):
"""加载现有的SAGA"""
saga_ids = self.persistence.get_all_sagas()
for saga_id in saga_ids:
saga = self.persistence.load_saga(saga_id)
if saga:
self.sagas[saga_id] = saga
self.logger.info(f"加载现有SAGA: {saga_id}")
def create_saga(self, name: str) -> str:
"""创建SAGA"""
saga_id = super().create_saga(name)
saga = self.sagas[saga_id]
self.persistence.save_saga(saga)
return saga_id
def execute_saga(self, saga_id: str) -> bool:
"""执行SAGA"""
try:
result = super().execute_saga(saga_id)
saga = self.sagas.get(saga_id)
if saga:
if result and saga.status == SagaStatus.COMPLETED:
# 成功完成，删除持久化状态
self.persistence.delete_saga(saga_id)
else:
# 保存状态
self.persistence.save_saga(saga)
return result
except Exception as e:
self.logger.error(f"执行SAGA异常: {saga_id}, {e}")
saga = self.sagas.get(saga_id)
if saga:
self.persistence.save_saga(saga)
return False
def _execute_step(self, saga: Saga, step: SagaStep) -> bool:
"""执行步骤并保存状态"""
result = super()._execute_step(saga, step)
self.persistence.save_saga(saga)
return result
def _compensate_step(self, saga: Saga, step: SagaStep) -> bool:
"""补偿步骤并保存状态"""
result = super()._compensate_step(saga, step)
self.persistence.save_saga(saga)
return result
def recover_unfinished_sagas(self) -> int:
"""恢复未完成的SAGA"""
unfinished_saga_ids = self.persistence.get_unfinished_sagas()
recovered_count = 0
for saga_id in unfinished_saga_ids:
saga = self.sagas.get(saga_id)
if not saga:
continue
self.logger.info(f"恢复SAGA: {saga_id}, 状态: {saga.status.value}")
try:
if saga.status == SagaStatus.RUNNING:
# 继续执行
if self._continue_saga(saga):
recovered_count += 1
elif saga.status == SagaStatus.COMPENSATING:
# 继续补偿
if self._continue_compensation(saga):
recovered_count += 1
# 更新持久化状态
if saga.status == SagaStatus.COMPLETED:
self.persistence.delete_saga(saga_id)
else:
self.persistence.save_saga(saga)
except Exception as e:
self.logger.error(f"恢复SAGA失败: {saga_id}, {e}")
self.logger.info(f"恢复完成，成功恢复 {recovered_count} 个SAGA")
return recovered_count
def _continue_saga(self, saga: Saga) -> bool:
"""继续执行SAGA"""
# 找到下一个需要执行的步骤
next_step_index = -1
for i, step in enumerate(saga.steps):
if step.status == SagaStepStatus.PENDING:
next_step_index = i
break
elif step.status == SagaStepStatus.RUNNING:
# 重新执行运行中的步骤
next_step_index = i
break
if next_step_index == -1:
# 所有步骤都已完成，标记为完成
saga.status = SagaStatus.COMPLETED
saga.updated_at = time.time()
return True
# 执行剩余步骤
for i in range(next_step_index, len(saga.steps)):
step = saga.steps[i]
saga.current_step_index = i
if not self._execute_step(saga, step):
# 步骤失败，开始补偿
self._compensate_saga(saga)
return False
# 所有步骤成功
saga.status = SagaStatus.COMPLETED
saga.updated_at = time.time()
return True
def _continue_compensation(self, saga: Saga) -> bool:
"""继续补偿SAGA"""
# 找到下一个需要补偿的步骤
completed_steps = [step for step in saga.steps if step.status == SagaStepStatus.COMPLETED]
compensated_steps = [step for step in saga.steps if step.status == SagaStepStatus.COMPENSATED]
# 补偿未补偿的已完成步骤
for step in reversed(completed_steps):
if step.status == SagaStepStatus.COMPLETED:
if not self._compensate_step(saga, step):
# 补偿失败
saga.error = f"补偿步骤失败: {step.name}"
saga.status = SagaStatus.FAILED
saga.updated_at = time.time()
return False
# 所有补偿完成
saga.status = SagaStatus.COMPENSATED
saga.updated_at = time.time()
return True
def get_saga_statistics(self) -> Dict:
"""获取SAGA统计信息"""
total_sagas = len(self.sagas)
if total_sagas == 0:
return {
'total': 0,
'running': 0,
'completed': 0,
'compensating': 0,
'compensated': 0,
'failed': 0
}
status_count = {
SagaStatus.RUNNING: 0,
SagaStatus.COMPLETED: 0,
SagaStatus.COMPENSATING: 0,
SagaStatus.COMPENSATED: 0,
SagaStatus.FAILED: 0
}
for saga in self.sagas.values():
status_count[saga.status] += 1
return {
'total': total_sagas,
'running': status_count[SagaStatus.RUNNING],
'completed': status_count[SagaStatus.COMPLETED],
'compensating': status_count[SagaStatus.COMPENSATING],
'compensated': status_count[SagaStatus.COMPENSATED],
'failed': status_count[SagaStatus.FAILED]
}
# 使用示例
def persistent_saga_example():
"""持久化SAGA模式示例"""
logging.basicConfig(level=logging.INFO)
# 创建服务实例
order_service = OrderService()
payment_service = PaymentService()
inventory_service = InventoryService()
# 创建持久化编排器
orchestrator = PersistentSagaOrchestrator("./saga_storage")
# 模拟系统重启后恢复
print("检查未完成的SAGA...")
recovered_count = orchestrator.recover_unfinished_sagas()
print(f"恢复了 {recovered_count} 个未完成的SAGA")
# 显示统计信息
stats = orchestrator.get_saga_statistics()
print(f"SAGA统计: {stats}")
# 创建新的订单处理SAGA
saga_id = orchestrator.create_saga("持久化订单处理流程")
# 添加步骤：创建订单
orchestrator.add_step(
saga_id=saga_id,
name="创建订单",
execute_func=order_service.create_order,
compensate_func=order_service.cancel_order,
params={
'user_id': 'user001',
'items': ['laptop', 'keyboard'],
'total_amount': 4999.99
}
)
# 添加步骤：处理支付
orchestrator.add_step(
saga_id=saga_id,
name="处理支付",
execute_func=payment_service.process_payment,
compensate_func=payment_service.refund_payment,
params={
'user_id': 'user001',
'merchant_id': 'merchant001',
'amount': 4999.99
}
)
# 添加步骤：预留库存
orchestrator.add_step(
saga_id=saga_id,
name="预留库存",
execute_func=inventory_service.reserve_inventory,
compensate_func=inventory_service.release_inventory,
params={
'items': [
{'product_id': 'laptop', 'quantity': 1},
{'product_id': 'keyboard', 'quantity': 1}
]
}
)
print(f"\n执行前状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"库存 - 笔记本: {inventory_service.inventory['laptop']}, 键盘: {inventory_service.inventory['keyboard']}")
# 模拟执行到一半系统崩溃
print("\n模拟执行到一半...")
# 可以在这里设置断点或手动中断来测试恢复功能
# 执行SAGA
success = orchestrator.execute_saga(saga_id)
# 获取状态
status = orchestrator.get_saga_status(saga_id)
print(f"\n执行结果: {'成功' if success else '失败'}")
print(f"SAGA状态: {status['status']}")
print(f"\n执行后状态:")
print(f"用户余额: {payment_service.accounts['user001']}")
print(f"商家余额: {payment_service.accounts['merchant001']}")
print(f"库存 - 笔记本: {inventory_service.inventory['laptop']}, 键盘: {inventory_service.inventory['keyboard']}")
# 模拟系统重启后再次恢复
print("\n模拟系统重启后恢复...")
new_orchestrator = PersistentSagaOrchestrator("./saga_storage")
recovered_count = new_orchestrator.recover_unfinished_sagas()
print(f"再次恢复了 {recovered_count} 个未完成的SAGA")
---
06.性能优化
a.异步执行
使用异步并发执行SAGA步骤，减少总执行时间。对于无依赖关系的步骤可以并行执行，提高系统吞吐量。
b.事件驱动
基于事件驱动架构实现SAGA协调，减少中央协调器的压力。使用消息队列实现松耦合的服务通信。
c.批量处理
对于相似的SAGA请求可以进行批量处理，减少网络开销和系统资源消耗。合理设置批量大小平衡性能和延迟。
6.5 最终一致性
01.最终一致性概述
a.基本概念
最终一致性是一种弱一致性模型，允许系统在短时间内存在数据不一致的状态，但保证在没有新更新的情况下，数据最终会达到一致状态。适用于对一致性要求不高但可用性要求较高的分布式系统。
b.CAP定理
根据CAP定理，分布式系统无法同时满足一致性（Consistency）、可用性（Availability）和分区容错性（Partition Tolerance）。最终一致性选择AP，即保证可用性和分区容错性，牺牲强一致性。
c.适用场景
适用于社交媒体、内容分发、缓存系统、电商库存管理等对实时性要求不高但对系统可用性要求高的场景。
02.一致性模型
a.强一致性
所有节点的数据在任何时刻都是完全一致的，读取操作总是返回最新写入的数据。需要使用分布式锁或同步机制保证数据一致性，但会影响系统性能和可用性。
b.弱一致性
允许系统在一段时间内存在数据不一致，不同节点可能返回不同的数据。提高了系统的可用性和性能，但需要业务逻辑能够容忍暂时的数据不一致。
c.最终一致性
弱一致性的一种特殊形式，保证在没有新更新的情况下，系统最终会达到一致状态。通过异步复制、事件驱动等机制实现数据同步。
03.同步机制
a.异步复制
主节点接收写操作后立即返回，异步地将数据复制到从节点。提高了写操作的性能，但存在数据丢失的风险。
b.事件驱动
通过事件消息触发数据同步，当数据发生变化时发送事件通知其他节点。事件驱动的系统能够更好地处理复杂的业务逻辑。
c.定期同步
按照固定时间间隔进行数据同步，适用于对实时性要求不高的场景。简单的同步机制，但可能产生较长的延迟。
04.冲突解决
a.版本控制
为每个数据项分配版本号或时间戳，解决并发修改冲突。使用向量时钟或Lamport时钟等算法确定操作顺序。
b.业务规则
根据业务逻辑制定冲突解决策略，如最后写入优先、业务规则优先等。需要根据具体业务场景设计合理的冲突解决机制。
c.人工介入
当自动冲突解决无法处理时，需要人工介入进行手动调整。提供冲突报告和解决工具帮助运维人员处理复杂冲突。
05.实现示例
a.基础最终一致性
---
# 基础最终一致性实现
import logging
import time
import uuid
from typing import Dict, List, Optional, Any, Callable
from enum import Enum
from dataclasses import dataclass, field
from collections import defaultdict
import threading
import queue
class ConsistencyLevel(Enum):
"""一致性级别"""
EVENTUAL = "EVENTUAL"
STRONG = "STRONG"
SESSION = "SESSION"
MONOTONIC = "MONOTONIC"
class DataOperation:
"""数据操作"""
def __init__(self, operation_type: str, key: str, value: Any, timestamp: float = None):
self.operation_id = str(uuid.uuid4())
self.operation_type = operation_type # 'create', 'update', 'delete'
self.key = key
self.value = value
self.timestamp = timestamp or time.time()
self.node_id = None
class DataNode:
"""数据节点"""
def __init__(self, node_id: str, consistency_level: ConsistencyLevel = ConsistencyLevel.EVENTUAL):
self.node_id = node_id
self.consistency_level = consistency_level
self.data = {}
self.operation_queue = queue.Queue()
self.pending_operations = {}
self.vector_clock = defaultdict(int)
self.logger = logging.getLogger(f"DataNode-{node_id}")
self.running = False
def start(self):
"""启动节点"""
self.running = True
self.process_thread = threading.Thread(target=self._process_operations)
self.process_thread.daemon = True
self.process_thread.start()
self.logger.info(f"节点启动: {self.node_id}")
def stop(self):
"""停止节点"""
self.running = False
if hasattr(self, 'process_thread'):
self.process_thread.join()
self.logger.info(f"节点停止: {self.node_id}")
def write(self, key: str, value: Any) -> bool:
"""写入数据"""
operation = DataOperation('update', key, value)
operation.node_id = self.node_id
# 更新向量时钟
self.vector_clock[self.node_id] += 1
operation.vector_clock = dict(self.vector_clock)
if self.consistency_level == ConsistencyLevel.STRONG:
# 强一致性：立即执行
return self._execute_operation(operation)
else:
# 最终一致性：加入队列
self.operation_queue.put(operation)
self.pending_operations[operation.operation_id] = operation
return True
def read(self, key: str) -> Optional[Any]:
"""读取数据"""
return self.data.get(key)
def replicate_operation(self, operation: DataOperation):
"""复制操作"""
self.operation_queue.put(operation)
def _process_operations(self):
"""处理操作队列"""
while self.running:
try:
# 从队列获取操作
operation = self.operation_queue.get(timeout=1)
self._execute_operation(operation)
self.operation_queue.task_done()
except queue.Empty:
continue
except Exception as e:
self.logger.error(f"处理操作异常: {e}")
def _execute_operation(self, operation: DataOperation) -> bool:
"""执行操作"""
try:
if operation.operation_type == 'create':
if operation.key not in self.data:
self.data[operation.key] = operation.value
self.logger.info(f"创建数据: {operation.key}")
return True
elif operation.operation_type == 'update':
# 检查向量时钟
if self._should_apply_operation(operation):
self.data[operation.key] = operation.value
self._update_vector_clock(operation)
self.logger.info(f"更新数据: {operation.key}")
return True
elif operation.operation_type == 'delete':
if operation.key in self.data:
del self.data[operation.key]
self.logger.info(f"删除数据: {operation.key}")
return True
except Exception as e:
self.logger.error(f"执行操作失败: {e}")
return False
def _should_apply_operation(self, operation: DataOperation) -> bool:
"""判断是否应该应用操作"""
if not hasattr(operation, 'vector_clock'):
return True
# 比较向量时钟
for node_id, timestamp in operation.vector_clock.items():
if timestamp > self.vector_clock.get(node_id, 0):
return True
return False
def _update_vector_clock(self, operation: DataOperation):
"""更新向量时钟"""
if hasattr(operation, 'vector_clock'):
for node_id, timestamp in operation.vector_clock.items():
self.vector_clock[node_id] = max(self.vector_clock.get(node_id, 0), timestamp)
def get_status(self) -> Dict:
"""获取节点状态"""
return {
'node_id': self.node_id,
'consistency_level': self.consistency_level.value,
'data_count': len(self.data),
'pending_operations': len(self.pending_operations),
'vector_clock': dict(self.vector_clock),
'queue_size': self.operation_queue.qsize()
}
class EventualConsistencyCluster:
"""最终一致性集群"""
def __init__(self):
self.nodes = {}
self.replication_factor = 2 # 复制因子
self.logger = logging.getLogger("EventualConsistencyCluster")
def add_node(self, node: DataNode):
"""添加节点"""
self.nodes[node.node_id] = node
self.logger.info(f"添加节点: {node.node_id}")
def start_cluster(self):
"""启动集群"""
for node in self.nodes.values():
node.start()
self.logger.info("集群启动完成")
def stop_cluster(self):
"""停止集群"""
for node in self.nodes.values():
node.stop()
self.logger.info("集群停止完成")
def write(self, key: str, value: Any) -> bool:
"""写入数据（复制到多个节点）"""
# 选择要复制的节点
target_nodes = self._select_replication_nodes()
success_count = 0
for node in target_nodes:
if node.write(key, value):
success_count += 1
# 异步复制到其他节点
self._replicate_to_other_nodes(key, value, target_nodes)
return success_count >= 1 # 至少一个节点成功
def read(self, key: str) -> Optional[Any]:
"""读取数据（从任意节点）"""
node_ids = list(self.nodes.keys())
if not node_ids:
return None
# 随机选择一个节点读取
import random
node_id = random.choice(node_ids)
return self.nodes[node_id].read(key)
def _select_replication_nodes(self) -> List[DataNode]:
"""选择复制节点"""
node_list = list(self.nodes.values())
import random
if len(node_list) <= self.replication_factor:
return node_list
return random.sample(node_list, self.replication_factor)
def _replicate_to_other_nodes(self, key: str, value: Any, exclude_nodes: List[DataNode]):
"""复制到其他节点"""
exclude_ids = {node.node_id for node in exclude_nodes}
other_nodes = [node for node in self.nodes.values() if node.node_id not in exclude_ids]
for node in other_nodes:
# 异步复制
import threading
thread = threading.Thread(target=node.write, args=(key, value))
thread.daemon = True
thread.start()
def check_consistency(self, key: str) -> Dict[str, Any]:
"""检查一致性"""
results = {}
for node_id, node in self.nodes.items():
value = node.read(key)
results[node_id] = {
'value': value,
'vector_clock': dict(node.vector_clock)
}
return {
'key': key,
'consistent': len(set(r['value'] for r in results.values() if r['value'] is not None)) <= 1,
'nodes': results
}
def get_cluster_status(self) -> Dict:
"""获取集群状态"""
return {
'node_count': len(self.nodes),
'replication_factor': self.replication_factor,
'nodes': {node_id: node.get_status() for node_id, node in self.nodes.items()}
}
# 使用示例
def basic_eventual_consistency_example():
"""基础最终一致性示例"""
logging.basicConfig(level=logging.INFO)
# 创建集群
cluster = EventualConsistencyCluster()
# 创建节点
node1 = DataNode("node1", ConsistencyLevel.EVENTUAL)
node2 = DataNode("node2", ConsistencyLevel.EVENTUAL)
node3 = DataNode("node3", ConsistencyLevel.EVENTUAL)
# 添加节点到集群
cluster.add_node(node1)
cluster.add_node(node2)
cluster.add_node(node3)
# 启动集群
cluster.start_cluster()
# 写入数据
print("写入数据...")
cluster.write("user:001", {"name": "张三", "age": 25, "city": "北京"})
cluster.write("user:002", {"name": "李四", "age": 30, "city": "上海"})
# 等待复制完成
time.sleep(2)
# 读取数据
print("\n读取数据:")
print(f"用户001: {cluster.read('user:001')}")
print(f"用户002: {cluster.read('user:002')}")
# 检查一致性
print("\n检查一致性:")
consistency_check = cluster.check_consistency("user:001")
print(f"数据一致性: {consistency_check['consistent']}")
for node_id, info in consistency_check['nodes'].items():
print(f"节点 {node_id}: {info['value']}")
# 获取集群状态
print("\n集群状态:")
status = cluster.get_cluster_status()
for node_id, node_status in status['nodes'].items():
print(f"节点 {node_id}: {node_status['data_count']} 条数据")
# 停止集群
cluster.stop_cluster()
---
b.事件驱动同步
---
# 事件驱动的最终一致性实现
import asyncio
import json
from typing import Dict, List, Optional, Callable
from dataclasses import dataclass, asdict
from enum import Enum
class EventType(Enum):
"""事件类型"""
DATA_CREATED = "DATA_CREATED"
DATA_UPDATED = "DATA_UPDATED"
DATA_DELETED = "DATA_DELETED"
SYNC_REQUEST = "SYNC_REQUEST"
SYNC_RESPONSE = "SYNC_RESPONSE"
@dataclass
class DataEvent:
"""数据事件"""
event_id: str
event_type: EventType
key: str
value: Any
timestamp: float
source_node: str
target_nodes: List[str] = field(default_factory=list)
metadata: Dict = field(default_factory=dict)
class EventualConsistencyNode:
"""事件驱动的一致性节点"""
def __init__(self, node_id: str, event_bus: 'EventBus'):
self.node_id = node_id
self.event_bus = event_bus
self.data = {}
self.pending_events = {}
self.event_handlers = {}
self.logger = logging.getLogger(f"EventualNode-{node_id}")
self._setup_event_handlers()
def _setup_event_handlers(self):
"""设置事件处理器"""
self.event_handlers[EventType.DATA_CREATED] = self._handle_data_created
self.event_handlers[EventType.DATA_UPDATED] = self._handle_data_updated
self.event_handlers[EventType.DATA_DELETED] = self._handle_data_deleted
self.event_handlers[EventType.SYNC_REQUEST] = self._handle_sync_request
self.event_handlers[EventType.SYNC_RESPONSE] = self._handle_sync_response
async def create_data(self, key: str, value: Any) -> bool:
"""创建数据"""
if key in self.data:
return False
self.data[key] = {
'value': value,
'created_at': time.time(),
'updated_at': time.time(),
'version': 1
}
# 发布创建事件
event = DataEvent(
event_id=str(uuid.uuid4()),
event_type=EventType.DATA_CREATED,
key=key,
value=value,
timestamp=time.time(),
source_node=self.node_id
)
await self.event_bus.publish(event)
self.logger.info(f"创建数据: {key}")
return True
async def update_data(self, key: str, value: Any) -> bool:
"""更新数据"""
if key not in self.data:
return False
old_value = self.data[key]['value']
self.data[key]['value'] = value
self.data[key]['updated_at'] = time.time()
self.data[key]['version'] += 1
# 发布更新事件
event = DataEvent(
event_id=str(uuid.uuid4()),
event_type=EventType.DATA_UPDATED,
key=key,
value=value,
timestamp=time.time(),
source_node=self.node_id,
metadata={'old_value': old_value, 'version': self.data[key]['version']}
)
await self.event_bus.publish(event)
self.logger.info(f"更新数据: {key}")
return True
async def delete_data(self, key: str) -> bool:
"""删除数据"""
if key not in self.data:
return False
old_value = self.data[key]['value']
del self.data[key]
# 发布删除事件
event = DataEvent(
event_id=str(uuid.uuid4()),
event_type=EventType.DATA_DELETED,
key=key,
value=old_value,
timestamp=time.time(),
source_node=self.node_id
)
await self.event_bus.publish(event)
self.logger.info(f"删除数据: {key}")
return True
def read_data(self, key: str) -> Optional[Any]:
"""读取数据"""
if key in self.data:
return self.data[key]['value']
return None
async def request_sync(self, target_node: str):
"""请求同步"""
event = DataEvent(
event_id=str(uuid.uuid4()),
event_type=EventType.SYNC_REQUEST,
key="",
value={},
timestamp=time.time(),
source_node=self.node_id,
target_nodes=[target_node]
)
await self.event_bus.publish(event)
self.logger.info(f"请求同步: {target_node}")
async def _handle_data_created(self, event: DataEvent):
"""处理数据创建事件"""
if event.source_node == self.node_id:
return # 忽略自己发送的事件
if event.key not in self.data:
self.data[event.key] = {
'value': event.value,
'created_at': event.timestamp,
'updated_at': event.timestamp,
'version': 1,
'synced_from': event.source_node
}
self.logger.info(f"同步创建数据: {event.key} from {event.source_node}")
async def _handle_data_updated(self, event: DataEvent):
"""处理数据更新事件"""
if event.source_node == self.node_id:
return
if event.key in self.data:
# 版本检查
current_version = self.data[event.key].get('version', 0)
event_version = event.metadata.get('version', 0)
if event_version > current_version:
old_value = self.data[event.key]['value']
self.data[event.key]['value'] = event.value
self.data[event.key]['updated_at'] = event.timestamp
self.data[event.key]['version'] = event_version
self.data[event.key]['synced_from'] = event.source_node
self.logger.info(f"同步更新数据: {event.key} from {event.source_node}")
async def _handle_data_deleted(self, event: DataEvent):
"""处理数据删除事件"""
if event.source_node == self.node_id:
return
if event.key in self.data:
del self.data[event.key]
self.logger.info(f"同步删除数据: {event.key} from {event.source_node}")
async def _handle_sync_request(self, event: DataEvent):
"""处理同步请求"""
if self.node_id not in event.target_nodes:
return
# 发送同步响应
response_event = DataEvent(
event_id=str(uuid.uuid4()),
event_type=EventType.SYNC_RESPONSE,
key="",
value=self.data,
timestamp=time.time(),
source_node=self.node_id,
target_nodes=[event.source_node]
)
await self.event_bus.publish(response_event)
self.logger.info(f"响应同步请求: {event.source_node}")
async def _handle_sync_response(self, event: DataEvent):
"""处理同步响应"""
if self.node_id not in event.target_nodes:
return
# 合并数据
remote_data = event.value
for key, data_info in remote_data.items():
if key not in self.data:
self.data[key] = data_info.copy()
self.data[key]['synced_from'] = event.source_node
self.logger.info(f"完成数据同步: {event.source_node}")
def get_data_info(self) -> Dict:
"""获取数据信息"""
return {
'node_id': self.node_id,
'data_count': len(self.data),
'data_keys': list(self.data.keys()),
'data_summary': {
key: {
'value': info['value'],
'version': info.get('version', 0),
'synced_from': info.get('synced_from')
}
for key, info in self.data.items()
}
}
class EventBus:
"""事件总线"""
def __init__(self):
self.nodes = {}
self.events = []
self.logger = logging.getLogger("EventBus")
def register_node(self, node: EventualConsistencyNode):
"""注册节点"""
self.nodes[node.node_id] = node
self.logger.info(f"注册节点: {node.node_id}")
async def publish(self, event: DataEvent):
"""发布事件"""
self.events.append(event)
self.logger.info(f"发布事件: {event.event_type.value} - {event.key}")
# 通知所有相关节点
target_nodes = event.target_nodes if event.target_nodes else list(self.nodes.keys())
for node_id in target_nodes:
if node_id in self.nodes:
node = self.nodes[node_id]
handler = node.event_handlers.get(event.event_type)
if handler:
try:
await handler(event)
except Exception as e:
self.logger.error(f"事件处理异常: {e}")
# 使用示例
async def event_driven_consistency_example():
"""事件驱动一致性示例"""
logging.basicConfig(level=logging.INFO)
# 创建事件总线
event_bus = EventBus()
# 创建节点
node1 = EventualConsistencyNode("node1", event_bus)
node2 = EventualConsistencyNode("node2", event_bus)
node3 = EventualConsistencyNode("node3", event_bus)
# 注册节点
event_bus.register_node(node1)
event_bus.register_node(node2)
event_bus.register_node(node3)
# 在节点1创建数据
await node1.create_data("product:001", {
"name": "智能手机",
"price": 2999.99,
"stock": 100
})
# 在节点2创建数据
await node2.create_data("product:002", {
"name": "笔记本电脑",
"price": 5999.99,
"stock": 50
})
# 等待事件传播
await asyncio.sleep(1)
# 检查各节点的数据
print("节点1数据:")
info1 = node1.get_data_info()
for key, data in info1['data_summary'].items():
print(f" {key}: {data['value']} (来源: {data['synced_from'] or '本地'})")
print("\n节点2数据:")
info2 = node2.get_data_info()
for key, data in info2['data_summary'].items():
print(f" {key}: {data['value']} (来源: {data['synced_from'] or '本地'})")
print("\n节点3数据:")
info3 = node3.get_data_info()
for key, data in info3['data_summary'].items():
print(f" {key}: {data['value']} (来源: {data['synced_from'] or '本地'})")
# 更新数据
print("\n更新节点1的数据...")
await node1.update_data("product:001", {
"name": "智能手机",
"price": 2799.99,
"stock": 90
})
# 等待更新传播
await asyncio.sleep(1)
# 再次检查数据
print("\n更新后的节点1数据:")
info1 = node1.get_data_info()
for key, data in info1['data_summary'].items():
print(f" {key}: {data['value']} (版本: {data['version']})")
print("\n更新后的节点2数据:")
info2 = node2.get_data_info()
for key, data in info2['data_summary'].items():
print(f" {key}: {data['value']} (版本: {data['version']})")
# 请求同步
print("\n节点3请求同步...")
await node3.request_sync("node1")
# 等待同步完成
await asyncio.sleep(1)
print("\n同步后的节点3数据:")
info3 = node3.get_data_info()
for key, data in info3['data_summary'].items():
print(f" {key}: {data['value']} (版本: {data['version']})")
print(f"\n事件总数: {len(event_bus.events)}")
---
c.冲突检测和解决
---
# 冲突检测和解决的最终一致性
import hashlib
from typing import Dict, List, Optional, Tuple, Any
from dataclasses import dataclass, field
from enum import Enum
class ConflictResolutionStrategy(Enum):
"""冲突解决策略"""
LAST_WRITE_WINS = "LAST_WRITE_WINS"
FIRST_WRITE_WINS = "FIRST_WRITE_WINS"
MERGE = "MERGE"
BUSINESS_RULES = "BUSINESS_RULES"
MANUAL = "MANUAL"
@dataclass
class DataVersion:
"""数据版本"""
version_id: str
value: Any
timestamp: float
node_id: str
checksum: str
metadata: Dict = field(default_factory=dict)
@dataclass
class ConflictRecord:
"""冲突记录"""
conflict_id: str
key: str
versions: List[DataVersion]
detected_at: float
resolution_strategy: Optional[ConflictResolutionStrategy] = None
resolved_value: Optional[Any] = None
resolved_at: Optional[float] = None
class ConflictAwareNode:
"""冲突感知的一致性节点"""
def __init__(self, node_id: str):
self.node_id = node_id
self.data = {}
self.versions = {} # key -> List[DataVersion]
self.conflicts = {} # conflict_id -> ConflictRecord
self.resolution_strategies = {}
self.logger = logging.getLogger(f"ConflictAwareNode-{node_id}")
self._setup_resolution_strategies()
def _setup_resolution_strategies(self):
"""设置冲突解决策略"""
self.resolution_strategies[ConflictResolutionStrategy.LAST_WRITE_WINS] = self._resolve_last_write_wins
self.resolution_strategies[ConflictResolutionStrategy.FIRST_WRITE_WINS] = self._resolve_first_write_wins
self.resolution_strategies[ConflictResolutionStrategy.MERGE] = self._resolve_merge
self.resolution_strategies[ConflictResolutionStrategy.BUSINESS_RULES] = self._resolve_business_rules
def write_data(self, key: str, value: Any, timestamp: float = None) -> str:
"""写入数据"""
timestamp = timestamp or time.time()
checksum = self._calculate_checksum(value)
version_id = str(uuid.uuid4())
version = DataVersion(
version_id=version_id,
value=value,
timestamp=timestamp,
node_id=self.node_id,
checksum=checksum,
metadata={'operation': 'write'}
)
# 添加到版本历史
if key not in self.versions:
self.versions[key] = []
self.versions[key].append(version)
# 更新当前数据
self.data[key] = value
# 检测冲突
conflicts = self._detect_conflicts(key)
for conflict in conflicts:
self.conflicts[conflict.conflict_id] = conflict
self.logger.warning(f"检测到冲突: {key} - {conflict.conflict_id}")
return version_id
def replicate_version(self, version: DataVersion):
"""复制版本"""
key = self._get_key_for_version(version)
if key:
if key not in self.versions:
self.versions[key] = []
# 检查是否已存在相同版本
existing_version_ids = {v.version_id for v in self.versions[key]}
if version.version_id not in existing_version_ids:
self.versions[key].append(version)
version.metadata['operation'] = 'replication'
# 检测冲突
conflicts = self._detect_conflicts(key)
for conflict in conflicts:
self.conflicts[conflict.conflict_id] = conflict
self.logger.warning(f"检测到冲突: {key} - {conflict.conflict_id}")
def read_data(self, key: str) -> Optional[Any]:
"""读取数据"""
# 返回最新版本的数据
if key in self.data:
return self.data[key]
return None
def _detect_conflicts(self, key: str) -> List[ConflictRecord]:
"""检测冲突"""
conflicts = []
versions = self.versions.get(key, [])
if len(versions) < 2:
return conflicts
# 检查是否有不同的值
unique_values = {}
for version in versions:
checksum = version.checksum
if checksum not in unique_values:
unique_values[checksum] = []
unique_values[checksum].append(version)
if len(unique_values) <= 1:
return conflicts # 没有冲突
# 创建冲突记录
conflict_versions = []
for version_list in unique_values.values():
conflict_versions.extend(version_list)
conflict = ConflictRecord(
conflict_id=str(uuid.uuid4()),
key=key,
versions=conflict_versions,
detected_at=time.time()
)
conflicts.append(conflict)
return conflicts
def resolve_conflict(self, conflict_id: str, strategy: ConflictResolutionStrategy) -> bool:
"""解决冲突"""
if conflict_id not in self.conflicts:
return False
conflict = self.conflicts[conflict_id]
resolver = self.resolution_strategies.get(strategy)
if not resolver:
self.logger.error(f"未知的冲突解决策略: {strategy}")
return False
try:
resolved_value = resolver(conflict)
if resolved_value is not None:
# 应用解决后的值
self.data[conflict.key] = resolved_value
conflict.resolved_value = resolved_value
conflict.resolution_strategy = strategy
conflict.resolved_at = time.time()
self.logger.info(f"冲突已解决: {conflict_id} - {strategy.value}")
return True
except Exception as e:
self.logger.error(f"解决冲突失败: {e}")
return False
def _resolve_last_write_wins(self, conflict: ConflictRecord) -> Any:
"""最后写入优先策略"""
latest_version = max(conflict.versions, key=lambda v: v.timestamp)
return latest_version.value
def _resolve_first_write_wins(self, conflict: ConflictRecord) -> Any:
"""首先写入优先策略"""
earliest_version = min(conflict.versions, key=lambda v: v.timestamp)
return earliest_version.value
def _resolve_merge(self, conflict: ConflictRecord) -> Any:
"""合并策略"""
# 简单的字典合并示例
merged_value = {}
for version in conflict.versions:
if isinstance(version.value, dict):
merged_value.update(version.value)
else:
# 对于非字典类型，选择最新的
return self._resolve_last_write_wins(conflict)
return merged_value
def _resolve_business_rules(self, conflict: ConflictRecord) -> Any:
"""业务规则策略"""
key = conflict.key
# 示例业务规则
if key.startswith("product:"):
return self._resolve_product_conflict(conflict)
elif key.startswith("user:"):
return self._resolve_user_conflict(conflict)
else:
# 默认使用最后写入
return self._resolve_last_write_wins(conflict)
def _resolve_product_conflict(self, conflict: ConflictRecord) -> Any:
"""解决产品冲突"""
# 业务规则：选择价格最高的版本
if all(isinstance(v.value, dict) and 'price' in v.value for v in conflict.versions):
highest_price_version = max(conflict.versions, key=lambda v: v.value['price'])
return highest_price_version.value
return self._resolve_last_write_wins(conflict)
def _resolve_user_conflict(self, conflict: ConflictRecord) -> Any:
"""解决用户冲突"""
# 业务规则：合并信息，但保留最新的联系方式
if all(isinstance(v.value, dict) for v in conflict.versions):
merged = {}
latest_contact = None
latest_timestamp = 0
for version in conflict.versions:
# 合并基本信息
for key, value in version.value.items():
if key not in merged:
merged[key] = value
# 保留最新的联系方式
if 'contact' in version.value and version.timestamp > latest_timestamp:
latest_contact = version.value['contact']
latest_timestamp = version.timestamp
if latest_contact:
merged['contact'] = latest_contact
return merged
return self._resolve_last_write_wins(conflict)
def _calculate_checksum(self, value: Any) -> str:
"""计算校验和"""
value_str = json.dumps(value, sort_keys=True, ensure_ascii=False)
return hashlib.md5(value_str.encode()).hexdigest()
def _get_key_for_version(self, version: DataVersion) -> Optional[str]:
"""根据版本获取键"""
for key, versions in self.versions.items():
if version in versions:
return key
return None
def get_conflict_summary(self) -> Dict:
"""获取冲突摘要"""
total_conflicts = len(self.conflicts)
resolved_conflicts = len([c for c in self.conflicts.values() if c.resolved_at is not None])
unresolved_conflicts = total_conflicts - resolved_conflicts
strategy_counts = {}
for conflict in self.conflicts.values():
if conflict.resolution_strategy:
strategy = conflict.resolution_strategy.value
strategy_counts[strategy] = strategy_counts.get(strategy, 0) + 1
return {
'total_conflicts': total_conflicts,
'resolved_conflicts': resolved_conflicts,
'unresolved_conflicts': unresolved_conflicts,
'resolution_strategies': strategy_counts,
'conflict_details': [
{
'conflict_id': conflict.conflict_id,
'key': conflict.key,
'versions_count': len(conflict.versions),
'detected_at': conflict.detected_at,
'resolved': conflict.resolved_at is not None
}
for conflict in self.conflicts.values()
]
}
def get_data_versions(self, key: str) -> List[Dict]:
"""获取数据的所有版本"""
versions = self.versions.get(key, [])
return [
{
'version_id': v.version_id,
'node_id': v.node_id,
'timestamp': v.timestamp,
'checksum': v.checksum,
'operation': v.metadata.get('operation', 'unknown')
}
for v in versions
]
# 使用示例
def conflict_resolution_example():
"""冲突解决示例"""
logging.basicConfig(level=logging.INFO)
# 创建冲突感知节点
node1 = ConflictAwareNode("node1")
node2 = ConflictAwareNode("node2")
# 模拟并发写入冲突
print("模拟并发写入冲突...")
# 节点1写入数据
node1.write_data("product:001", {
"name": "智能手机",
"price": 2999.99,
"stock": 100
})
# 节点2写入相同键但不同值
time.sleep(0.1) # 确保时间戳不同
node2.write_data("product:001", {
"name": "智能手机",
"price": 2799.99, # 不同价格
"stock": 95 # 不同库存
})
# 复制版本到对方节点
for version in node1.versions.get("product:001", []):
node2.replicate_version(version)
for version in node2.versions.get("product:001", []):
node1.replicate_version(version)
# 检查冲突
print("\n节点1冲突摘要:")
summary1 = node1.get_conflict_summary()
print(f"总冲突数: {summary1['total_conflicts']}")
print(f"未解决冲突: {summary1['unresolved_conflicts']}")
print("\n节点2冲突摘要:")
summary2 = node2.get_conflict_summary()
print(f"总冲突数: {summary2['total_conflicts']}")
print(f"未解决冲突: {summary2['unresolved_conflicts']}")
# 解决冲突
print("\n解决冲突...")
for conflict_id in summary1['conflict_details']:
if not conflict_id['resolved']:
# 使用业务规则解决冲突
node1.resolve_conflict(conflict_id['conflict_id'], ConflictResolutionStrategy.BUSINESS_RULES)
for conflict_id in summary2['conflict_details']:
if not conflict_id['resolved']:
# 使用最后写入优先解决冲突
node2.resolve_conflict(conflict_id['conflict_id'], ConflictResolutionStrategy.LAST_WRITE_WINS)
# 检查解决结果
print("\n冲突解决后:")
print(f"节点1数据: {node1.read_data('product:001')}")
print(f"节点2数据: {node2.read_data('product:001')}")
# 显示版本历史
print("\n节点1版本历史:")
for version_info in node1.get_data_versions("product:001"):
print(f" 版本 {version_info['version_id'][:8]}: 节点 {version_info['node_id']}, 操作 {version_info['operation']}")
# 最终冲突摘要
print("\n最终冲突摘要:")
final_summary1 = node1.get_conflict_summary()
final_summary2 = node2.get_conflict_summary()
print(f"节点1: 已解决 {final_summary1['resolved_conflicts']}/{final_summary1['total_conflicts']}")
print(f"节点2: 已解决 {final_summary2['resolved_conflicts']}/{final_summary2['total_conflicts']}")
---
06.监控和调优
a.一致性监控
实时监控数据一致性状态，跟踪不一致的情况和恢复时间。设置合理的告警阈值，及时发现和处理一致性问题。
b.性能监控
监控系统的读写延迟、吞吐量等性能指标。分析性能瓶颈，优化数据同步策略和节点配置。
c.调优策略
根据业务需求调整一致性级别、复制因子、同步频率等参数。在不同场景下选择合适的一致性模型和同步机制。
7. Redis事务
7.1 MULTI与EXEC
01.事务命令概述
a.MULTI命令
标记事务的开始，后续执行的命令将被放入队列中等待执行。
b.EXEC命令
执行事务队列中的所有命令，返回所有命令的执行结果。
c.DISCARD命令
取消事务队列，清空所有已入队的命令，恢复到正常状态。
d.代码示例
---
# Redis事务基本示例
import redis
# 连接Redis
r = redis.Redis(host='localhost', port=6379, db=0)
# 清空测试数据
r.delete('account:alice', 'account:bob')
# 设置初始余额
r.set('account:alice', 1000)
r.set('account:bob', 500)
# 开始事务
pipe = r.pipeline()
try:
# 原子性转账操作：Alice转100给Bob
pipe.decr('account:alice', 100) # Alice余额减少
pipe.incr('account:bob', 100) # Bob余额增加
# 执行事务
results = pipe.execute()
print("转账成功，执行结果:", results)
# 验证余额
alice_balance = r.get('account:alice')
bob_balance = r.get('account:bob')
print(f"Alice余额: {alice_balance}, Bob余额: {bob_balance}")
except Exception as e:
print(f"事务执行失败: {e}")
# 使用MULTI/EXEC命令的方式
print("\n使用原生MULTI/EXEC命令:")
# 重新设置初始数据
r.set('account:alice', 1000)
r.set('account:bob', 500)
# 执行MULTI开始事务
r.execute_command('MULTI')
# 入队命令
r.execute_command('DECRBY', 'account:alice', 100)
r.execute_command('INCRBY', 'account:bob', 100)
# 执行事务
results = r.execute_command('EXEC')
print("MULTI/EXEC执行结果:", results)
---
02.Redis事务机制
a.命令队列化
a.入队过程
在MULTI执行后，所有命令会被放入队列而不立即执行。
b.执行原子性
EXEC执行时，队列中的所有命令按顺序连续执行，不会被其他客户端命令打断。
c.结果返回
EXEC返回所有命令执行结果的列表，保持与命令队列相同的顺序。
d.代码示例
---
# 演示命令队列化过程
import redis
import time
r = redis.Redis(host='localhost', port=6379, db=0)
r.delete('key1', 'key2', 'key3')
# 使用pipeline()模拟MULTI/EXEC
pipe = r.pipeline()
print("开始入队命令:")
# 这些命令会被放入队列，不会立即执行
pipe.set('key1', 'value1')
print(" - SET key1 已入队")
pipe.set('key2', 'value2')
print(" - SET key2 已入队")
pipe.incr('counter')
print(" - INCR counter 已入队")
# 此时数据还未被修改
print(f"\n入队后，执行前的数据状态:")
print(f" key1: {r.get('key1')}") # 应该为None
print(f" key2: {r.get('key2')}") # 应该为None
print(f" counter: {r.get('counter')}") # 应该为None
# 执行队列中的所有命令
print("\n执行事务:")
results = pipe.execute()
print(f"执行结果: {results}")
# 验证数据已修改
print(f"\n执行后的数据状态:")
print(f" key1: {r.get('key1')}")
print(f" key2: {r.get('key2')}")
print(f" counter: {r.get('counter')}")
---
b.事务状态
a.正常状态
客户端正常发送和接收命令。
b.事务状态
客户端进入事务模式，命令被排队但不执行。
c.执行状态
正在执行事务队列中的命令。
03.事务特性
a.原子性保证
a.整体执行
事务中的所有命令要么全部执行，要么全部不执行。
b.执行隔离
在EXEC执行期间，Redis不会执行其他客户端的命令。
c.失败处理
如果事务中有语法错误的命令，整个事务将不会执行。
d.代码示例
---
# 演示事务的原子性
import redis
import threading
import time
r = redis.Redis(host='localhost', port=6379, db=0)
def concurrent_client():
"""并发客户端，尝试修改事务中的键"""
time.sleep(0.1) # 稍等一下
print(f"并发客户端尝试修改键")
try:
r.set('key1', 'interrupted')
print(f"并发客户端设置成功")
except Exception as e:
print(f"并发客户端设置失败: {e}")
# 清空数据
r.delete('key1', 'key2')
# 启动并发线程
thread = threading.Thread(target=concurrent_client)
thread.start()
# 主线程执行事务
pipe = r.pipeline()
pipe.set('key1', 'original')
pipe.set('key2', 'original')
print("主线程开始执行事务")
results = pipe.execute()
print(f"事务执行完成，结果: {results}")
# 检查最终结果
final_key1 = r.get('key1')
final_key2 = r.get('key2')
print(f"最终 key1: {final_key1}, key2: {final_key2}")
thread.join()
# 演示语法错误导致整个事务失败
print("\n演示语法错误导致事务失败:")
pipe = r.pipeline()
pipe.set('key3', 'value3')
# 故意使用错误的命令语法
try:
pipe.execute_command('INVALID_COMMAND', 'arg')
pipe.set('key4', 'value4') # 这个命令也不会执行
results = pipe.execute()
print(f"事务执行结果: {results}")
except Exception as e:
print(f"事务执行失败: {e}")
print(f"检查数据: key3={r.get('key3')}, key4={r.get('key4')}")
---
04.使用场景
a.批量操作
a.批量数据更新
需要同时更新多个相关键值的场景。
b.批量操作示例
---
# 批量更新用户信息
import redis
import json
r = redis.Redis(host='localhost', port=6379, db=0)
def batch_update_user_info():
"""批量更新用户信息的完整示例"""
users_data = {
'user:1001': {'name': 'Alice', 'age': 25, 'score': 85},
'user:1002': {'name': 'Bob', 'age': 30, 'score': 92},
'user:1003': {'name': 'Carol', 'age': 28, 'score': 78}
}
# 使用事务批量更新
pipe = r.pipeline()
for user_key, user_info in users_data.items():
# 设置用户基本信息
pipe.hset(user_key, 'name', user_info['name'])
pipe.hset(user_key, 'age', user_info['age'])
pipe.hset(user_key, 'score', user_info['score'])
# 更新用户索引
pipe.sadd('users:active', user_key)
# 设置用户过期时间（示例：24小时）
pipe.expire(user_key, 86400)
# 执行所有操作
try:
results = pipe.execute()
print(f"批量更新完成，共执行 {len(results)} 个操作")
# 验证更新结果
for user_key in users_data.keys():
user_data = r.hgetall(user_key)
print(f"{user_key}: {user_data}")
except Exception as e:
print(f"批量更新失败: {e}")
batch_update_user_info()
---
c.数据迁移
在不同键之间转移数据，保证操作的原子性。
d.数据迁移示例
---
# 原子性数据迁移示例
def atomic_data_migration():
"""将用户数据从旧格式迁移到新格式"""
# 旧格式: user_old:123 -> "name:alice,age:25"
# 新格式: user_new:123 -> hash结构
old_key = 'user_old:123'
new_key = 'user_new:123'
# 设置旧数据
r.set(old_key, 'name:alice,age:25,score:85')
# 读取旧数据
old_data = r.get(old_key)
if old_data:
# 解析旧数据
parts = old_data.decode().split(',')
user_info = {}
for part in parts:
key, value = part.split(':')
user_info[key] = value
# 使用事务进行原子性迁移
pipe = r.pipeline()
# 设置新格式的数据
for key, value in user_info.items():
pipe.hset(new_key, key, value)
# 删除旧数据
pipe.delete(old_key)
# 更新迁移记录
pipe.sadd('migrated_users', '123')
# 执行迁移事务
try:
results = pipe.execute()
print(f"数据迁移成功: {results}")
# 验证迁移结果
print(f"旧数据是否存在: {r.exists(old_key)}")
print(f"新数据内容: {r.hgetall(new_key)}")
except Exception as e:
print(f"数据迁移失败: {e}")
# 迁移失败，旧数据仍然存在
print(f"旧数据仍存在: {r.get(old_key)}")
atomic_data_migration()
---
7.2 WATCH乐观锁
01.WATCH机制概述
a.乐观锁原理
WATCH命令实现乐观锁机制，在事务执行前监视一个或多个键的变化。
b.监控机制
a.键监控
对指定的键设置监控标记，记录键的版本或值。
b.变更检测
在EXEC执行前检查被监控键是否被其他客户端修改。
c.事务决定
根据监控结果决定是否执行事务或返回失败。
c.UNWATCH命令
取消对所有键的监控，清除乐观锁状态。
d.代码示例
---
# WATCH乐观锁基本示例
import redis
import threading
import time
r = redis.Redis(host='localhost', port=6379, db=0)
def optimistic_counter_update(counter_key, increment):
"""使用WATCH实现乐观的计数器更新"""
max_retries = 3
retry_count = 0
while retry_count < max_retries:
try:
# 监控计数器键
r.watch(counter_key)
# 获取当前值
current_value = int(r.get(counter_key) or 0)
print(f"当前计数器值: {current_value}")
# 开始事务
pipe = r.pipeline()
# 计算新值
new_value = current_value + increment
# 设置新值
pipe.set(counter_key, new_value)
# 执行事务
results = pipe.execute()
if results is not None:
print(f"更新成功! 新值: {new_value}")
return new_value
else:
print("事务失败，开始重试...")
retry_count += 1
except redis.WatchError:
print(f"检测到并发修改，第 {retry_count + 1} 次重试")
retry_count += 1
time.sleep(0.1) # 短暂等待后重试
except Exception as e:
print(f"更新失败: {e}")
break
print(f"达到最大重试次数 {max_retries}，更新失败")
return None
# 测试乐观锁
r.delete('counter')
r.set('counter', 100)
print("使用WATCH乐观锁更新计数器:")
optimistic_counter_update('counter', 50)
---
02.执行流程
a.WATCH阶段
a.监控设置
客户端执行WATCH命令指定需要监控的键。
b.版本记录
Redis记录被监控键的当前版本或值。
c.事务准备
进入MULTI状态，准备执行事务命令。
b.事务执行阶段
a.命令入队
在MULTI后执行的所有命令被放入队列。
b.监控检查
在EXEC执行前检查所有被监控键的变更状态。
c.执行决策
如果有键被修改则放弃执行，否则正常执行事务。
c.结果处理
a.执行成功
事务正常执行，返回所有命令的执行结果。
b.执行失败
事务被取消，返回nil表示因WATCH冲突而失败。
c.重试机制
客户端根据失败情况决定是否重试整个事务。
d.执行流程示例
---
# WATCH执行流程详细示例
def demonstrate_watch_flow():
"""演示WATCH的完整执行流程"""
account_key = 'account:balance'
r.delete(account_key)
r.set(account_key, 1000)
print("=== WATCH执行流程演示 ===")
# 步骤1: 监控键
print("\n步骤1: WATCH监控账户余额")
r.watch(account_key)
print(f"监控的键: {account_key}")
# 步骤2: 读取当前值
print("\n步骤2: 读取当前余额")
current_balance = int(r.get(account_key))
print(f"当前余额: {current_balance}")
# 步骤3: 开始事务
print("\n步骤3: 开始事务MULTI")
pipe = r.pipeline()
# 步骤4: 基于当前值计算新值并入队命令
print("\n步骤4: 计算新值并入队命令")
withdrawal = 200
new_balance = current_balance - withdrawal
print(f"取款金额: {withdrawal}, 新余额: {new_balance}")
pipe.set(account_key, new_balance)
# 步骤5: 执行事务
print("\n步骤5: 执行事务EXEC")
try:
results = pipe.execute()
if results is not None:
print(f"✅ 事务执行成功: {results}")
print(f"最终余额: {r.get(account_key)}")
else:
print("❌ 事务因冲突被取消")
except redis.WatchError:
print("❌ 检测到WatchError，键被其他客户端修改")
demonstrate_watch_flow()
---
03.冲突检测与处理
a.变更识别
a.写操作检测
任何对被监控键的写操作都会触发冲突。
b.版本比较
比较键的当前版本与WATCH时的记录版本。
c.时间窗口
在WATCH到EXEC之间的时间窗口内进行变更检测。
b.并发冲突示例
---
# 演示WATCH并发冲突检测
def concurrent_watch_demo():
"""演示WATCH在并发环境下的冲突检测"""
shared_key = 'shared:counter'
r.delete(shared_key)
r.set(shared_key, 100)
def worker_worker(worker_id):
"""工作线程，尝试并发更新共享计数器"""
success = False
attempts = 0
while not success and attempts < 5:
try:
# 监控共享键
r.watch(shared_key)
# 读取当前值
current = int(r.get(shared_key))
print(f"工作线程{worker_id}: 当前值 {current}")
# 模拟一些处理时间
time.sleep(0.01)
# 开始事务
pipe = r.pipeline()
increment = worker_id * 10
new_value = current + increment
pipe.set(shared_key, new_value)
# 执行事务
results = pipe.execute()
if results is not None:
print(f"✅ 工作线程{worker_id}: 成功更新为 {new_value}")
success = True
else:
print(f"⚠️ 工作线程{worker_id}: 检测到冲突，准备重试")
attempts += 1
except redis.WatchError:
print(f"❌ 工作线程{worker_id}: WatchError，重试中...")
attempts += 1
except Exception as e:
print(f"❌ 工作线程{worker_id}: 错误 {e}")
break
if not success:
print(f"❌ 工作线程{worker_id}: 放弃重试")
# 启动多个并发工作线程
import threading
threads = []
for i in range(3):
thread = threading.Thread(target=worker_worker, args=(i+1,))
threads.append(thread)
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
print(f"\n最终共享计数器值: {r.get(shared_key)}")
print("=== 并发冲突检测演示 ===")
concurrent_watch_demo()
---
c.高级重试策略
---
# 高级重试策略示例
import random
import time
class OptimisticLockWithRetry:
def __init__(self, redis_client):
self.r = redis_client
self.max_retries = 5
self.base_delay = 0.01 # 10ms基础延迟
self.max_delay = 1.0 # 最大延迟1秒
def exponential_backoff(self, attempt):
"""指数退避算法"""
delay = min(self.base_delay * (2 ** attempt) + random.uniform(0, 0.1), self.max_delay)
time.sleep(delay)
return delay
def update_with_condition(self, key, condition_func, update_func):
"""带条件检查的乐观更新"""
attempt = 0
while attempt < self.max_retries:
try:
# 监控键
self.r.watch(key)
# 获取当前值
current_value = self.r.get(key)
# 检查条件
if not condition_func(current_value):
print(f"条件不满足，跳过更新")
self.r.unwatch()
return False
# 计算新值
new_value = update_func(current_value)
# 开始事务
pipe = self.r.pipeline()
pipe.set(key, new_value)
# 执行事务
results = pipe.execute()
if results is not None:
print(f"更新成功: {current_value} -> {new_value}")
return True
else:
attempt += 1
if attempt < self.max_retries:
delay = self.exponential_backoff(attempt)
print(f"冲突重试 {attempt}/{self.max_retries}，等待 {delay:.3f}s")
except redis.WatchError:
attempt += 1
if attempt < self.max_retries:
delay = self.exponential_backoff(attempt)
print(f"WatchError重试 {attempt}/{self.max_retries}，等待 {delay:.3f}s")
except Exception as e:
print(f"更新失败: {e}")
self.r.unwatch()
return False
print(f"达到最大重试次数，更新失败")
return False
# 使用高级重试策略
lock_manager = OptimisticLockWithRetry(r)
# 示例：只有当值小于1000时才增加
def condition_func(value):
return int(value or 0) < 1000
def update_func(value):
return int(value or 0) + 100
r.delete('conditional_counter')
r.set('conditional_counter', 800)
print("\n=== 高级重试策略演示 ===")
lock_manager.update_with_condition('conditional_counter', condition_func, update_func)
---
04.应用场景
a.库存管理
a.高并发库存扣减
在电商秒杀场景下安全地扣减商品库存。
b.库存管理示例
---
# 电商库存管理示例
class InventoryManager:
def __init__(self, redis_client):
self.r = redis_client
def deduct_inventory(self, product_id, quantity, user_id):
"""扣减库存，使用乐观锁保证并发安全"""
inventory_key = f'inventory:product:{product_id}'
order_key = f'orders:user:{user_id}'
max_retries = 3
retry_count = 0
while retry_count < max_retries:
try:
# 监控库存键
self.r.watch(inventory_key)
# 获取当前库存
current_stock = int(self.r.get(inventory_key) or 0)
# 检查库存是否充足
if current_stock < quantity:
print(f"库存不足，当前库存: {current_stock}, 需要: {quantity}")
self.r.unwatch()
return False, "库存不足"
# 开始事务
pipe = self.r.pipeline()
# 扣减库存
new_stock = current_stock - quantity
pipe.set(inventory_key, new_stock)
# 记录订单
pipe.hset(order_key, f'product:{product_id}', quantity)
pipe.hset(order_key, 'timestamp', int(time.time()))
# 执行事务
results = pipe.execute()
if results is not None:
print(f"✅ 用户{user_id} 成功购买 {quantity} 件商品{product_id}")
print(f"剩余库存: {new_stock}")
return True, f"购买成功，剩余库存: {new_stock}"
else:
retry_count += 1
print(f"⚠️ 库存冲突，第 {retry_count} 次重试")
except redis.WatchError:
retry_count += 1
print(f"⚠️ 检测到并发修改，第 {retry_count} 次重试")
if retry_count < max_retries:
time.sleep(0.05 * retry_count) # 递增延迟
except Exception as e:
print(f"❌ 库存扣减失败: {e}")
self.r.unwatch()
return False, f"系统错误: {e}"
print(f"❌ 达到最大重试次数，购买失败")
return False, "系统繁忙，请稍后重试"
# 测试库存管理
inventory_manager = InventoryManager(r)
# 设置初始库存
product_id = 1001
r.delete(f'inventory:product:{product_id}')
r.set(f'inventory:product:{product_id}', 50) # 50件商品
print("=== 电商库存管理演示 ===")
# 模拟多个用户并发购买
def simulate_user_purchase(user_id, quantity):
success, message = inventory_manager.deduct_inventory(product_id, quantity, user_id)
print(f"用户{user_id}: {message}")
return success
# 模拟10个用户并发购买
import threading
threads = []
for user_id in range(1, 11):
quantity = 5 if user_id <= 9 else 10 # 前9个用户买5件，第10个用户买10件
thread = threading.Thread(target=simulate_user_purchase, args=(user_id, quantity))
threads.append(thread)
thread.start()
# 等待所有购买完成
for thread in threads:
thread.join()
final_stock = r.get(f'inventory:product:{product_id}')
print(f"\n最终库存: {final_stock}")
---
b.限流控制
a.滑动窗口限流
使用WATCH实现基于滑动窗口的访问频率限制。
b.限流控制示例
---
# 基于WATCH的滑动窗口限流器
class RateLimiter:
def __init__(self, redis_client, window_size=60, max_requests=100):
self.r = redis_client
self.window_size = window_size # 时间窗口（秒）
self.max_requests = max_requests # 最大请求数
def is_allowed(self, identifier):
"""检查标识符是否允许访问"""
key = f'rate_limit:{identifier}'
current_time = int(time.time())
window_start = current_time - self.window_size
max_retries = 3
retry_count = 0
while retry_count < max_retries:
try:
# 监控限流键
self.r.watch(key)
# 获取当前窗口内的请求记录
requests = self.r.zrangebyscore(key, window_start, current_time)
# 计算当前请求数
request_count = len(requests)
print(f"标识符 {identifier}: 当前请求数 {request_count}/{self.max_requests}")
# 检查是否超过限制
if request_count >= self.max_requests:
self.r.unwatch()
print(f"❌ 标识符 {identifier} 超过限流阈值")
return False
# 开始事务：添加新请求记录并清理过期记录
pipe = self.r.pipeline()
# 添加新请求
pipe.zadd(key, {str(current_time): current_time})
# 删除过期记录
pipe.zremrangebyscore(key, 0, window_start)
# 设置过期时间
pipe.expire(key, self.window_size)
# 执行事务
results = pipe.execute()
if results is not None:
print(f"✅ 标识符 {identifier} 请求被允许")
return True
else:
retry_count += 1
print(f"⚠️ 限流检查冲突，第 {retry_count} 次重试")
except redis.WatchError:
retry_count += 1
print(f"⚠️ 限流检测到并发修改，第 {retry_count} 次重试")
if retry_count < max_retries:
time.sleep(0.01 * retry_count)
except Exception as e:
print(f"❌ 限流检查失败: {e}")
self.r.unwatch()
return False
print(f"❌ 标识符 {identifier} 限流检查达到最大重试次数")
return False
# 测试限流器
rate_limiter = RateLimiter(r, window_size=10, max_requests=5) # 10秒内最多5次请求
print("\n=== 滑动窗口限流演示 ===")
# 模拟同一用户快速发送多个请求
def simulate_requests(user_id, request_count):
allowed_count = 0
denied_count = 0
for i in range(request_count):
if rate_limiter.is_allowed(f"user:{user_id}"):
allowed_count += 1
else:
denied_count += 1
time.sleep(0.1) # 每100ms发送一个请求
print(f"用户{user_id}: 允许 {allowed_count} 次，拒绝 {denied_count} 次")
# 模拟用户请求
simulate_requests("alice", 15)
---
print(f"\n最终限流键状态:")
rate_limit_key = "rate_limit:user:alice"
if r.exists(rate_limit_key):
all_requests = r.zrange(rate_limit_key, 0, -1, withscores=True)
print(f"所有请求记录: {all_requests}")
else:
print("限流键不存在")
---
7.3 Pipeline批量操作
01.Pipeline机制概述
a.基本原理
Pipeline允许客户端一次性发送多个命令到Redis服务器，减少网络往返时间。
b.执行模式
a.批处理模式
将多个命令打包发送，服务器按顺序执行并返回结果。
b.非阻塞模式
命令在服务器端连续执行，提高整体处理效率。
c.性能优势
a.减少网络延迟
多个命令共享一个网络往返，显著减少网络开销。
b.提高吞吐量
在高并发场景下大幅提升Redis的每秒处理能力。
d.代码示例
---
# Pipeline基本使用示例
import redis
import time
r = redis.Redis(host='localhost', port=6379, db=0)
def compare_pipeline_performance():
"""对比Pipeline和普通命令的性能差异"""
# 清空测试数据
r.delete('perf:test1', 'perf:test2', 'perf:test3')
# 普通方式执行1000个命令
print("=== 普通命令执行方式 ===")
start_time = time.time()
for i in range(1000):
r.set(f'key:{i}', f'value:{i}')
normal_time = time.time() - start_time
print(f"普通方式执行1000个SET命令耗时: {normal_time:.4f}秒")
# 清空数据准备Pipeline测试
for i in range(1000):
r.delete(f'key:{i}')
# Pipeline方式执行1000个命令
print("\n=== Pipeline执行方式 ===")
start_time = time.time()
pipe = r.pipeline()
for i in range(1000):
pipe.set(f'key:{i}', f'value:{i}')
pipe.execute()
pipeline_time = time.time() - start_time
print(f"Pipeline方式执行1000个SET命令耗时: {pipeline_time:.4f}秒")
# 计算性能提升
improvement = (normal_time - pipeline_time) / normal_time * 100
print(f"\n性能提升: {improvement:.1f}% (Pipeline更快)")
compare_pipeline_performance()
---
02.Pipeline使用方式
a.基本Pipeline
a.创建Pipeline
通过redis.Redis().pipeline()创建pipeline对象。
b.命令入队
调用pipeline对象的方法，命令会被排队而不立即执行。
c.批量执行
调用execute()方法一次性执行所有排队的命令。
d.基本示例
---
# Pipeline基本使用示例
def basic_pipeline_usage():
"""演示Pipeline的基本使用方法"""
# 清空测试数据
r.delete('user:1:name', 'user:1:age', 'user:1:email')
r.delete('stats:users', 'stats:visits')
# 创建Pipeline
pipe = r.pipeline()
# 批量设置用户信息
pipe.set('user:1:name', 'Alice')
pipe.set('user:1:age', 25)
pipe.set('user:1:email', '[email protected]')
# 更新统计信息
pipe.incr('stats:users')
pipe.incrby('stats:visits', 3)
# 设置过期时间
pipe.expire('user:1:name', 86400) # 24小时
pipe.expire('user:1:age', 86400)
pipe.expire('user:1:email', 86400)
# 执行所有命令
results = pipe.execute()
print("Pipeline执行结果:")
commands = ['SET name', 'SET age', 'SET email', 'INCR users', 'INCRBY visits', 'EXPIRE name', 'EXPIRE age', 'EXPIRE email']
for i, (cmd, result) in enumerate(zip(commands, results)):
print(f" {cmd}: {result}")
# 验证数据
print(f"\n验证数据:")
print(f" 姓名: {r.get('user:1:name')}")
print(f" 年龄: {r.get('user:1:age')}")
print(f" 邮箱: {r.get('user:1:email')}")
print(f" 用户数: {r.get('stats:users')}")
print(f" 访问数: {r.get('stats:visits')}")
basic_pipeline_usage()
---
b.事务Pipeline
a.事务保证
使用transaction=True确保命令的原子性执行。
b.错误处理
事务模式下任何一个命令失败，整个Pipeline都不会执行。
c.事务示例
---
# 事务Pipeline示例
def transactional_pipeline():
"""演示事务Pipeline的使用"""
account_a = 'account:A'
account_b = 'account:B'
# 设置初始余额
r.set(account_a, 1000)
r.set(account_b, 500)
print("=== 事务Pipeline转账示例 ===")
print(f"转账前: A={r.get(account_a)}, B={r.get(account_b)}")
# 创建事务Pipeline
pipe = r.pipeline(transaction=True)
# 原子性转账操作
pipe.decrby(account_a, 200) # A减少200
pipe.incrby(account_b, 200) # B增加200
# 记录转账日志
pipe.lpush('transfer:log', f'A->B:200:{int(time.time())}')
try:
# 执行事务
results = pipe.execute()
print(f"事务执行结果: {results}")
print(f"转账后: A={r.get(account_a)}, B={r.get(account_b)}")
# 查看转账日志
logs = r.lrange('transfer:log', 0, -1)
print(f"转账日志: {logs}")
except Exception as e:
print(f"事务执行失败: {e}")
# 演示事务失败的情况
print("\n=== 演示事务失败 ===")
# 重置数据
r.set(account_a, 1000)
r.set(account_b, 500)
pipe = r.pipeline(transaction=True)
pipe.decrby(account_a, 200)
pipe.incrby(account_b, 200)
# 故意添加一个会导致错误的命令
pipe.execute_command('INVALID_COMMAND')
pipe.lpush('transfer:log', 'test')
try:
results = pipe.execute()
print(f"意外成功: {results}")
except Exception as e:
print(f"预期的事务失败: {e}")
print(f"数据未变化: A={r.get(account_a)}, B={r.get(account_b)}")
transactional_pipeline()
---
03.高级Pipeline技巧
a.批量读取优化
a.数据聚合
使用Pipeline批量读取多个键，然后进行数据聚合处理。
b.缓存预热
批量预加载热点数据到缓存中。
c.批量读取示例
---
# 批量读取和数据处理示例
def batch_read_and_process():
"""演示批量读取和数据处理"""
import json
import random
# 准备测试数据
print("=== 准备测试数据 ===")
products = {}
categories = ['electronics', 'clothing', 'books', 'home', 'sports']
for i in range(100):
product_id = f'product:{i+1:04d}'
product_data = {
'id': i+1,
'name': f'Product {i+1}',
'category': random.choice(categories),
'price': round(random.uniform(10, 1000), 2),
'stock': random.randint(0, 100),
'rating': round(random.uniform(1, 5), 1)
}
# 使用Pipeline批量写入产品数据
pipe = r.pipeline()
for key, value in product_data.items():
pipe.hset(product_id, key, value)
pipe.execute()
# 添加到分类索引
pipe = r.pipeline()
pipe.sadd(f'category:{product_data["category"]}', product_id)
pipe.zadd('products:by_price', {product_id: product_data["price"]})
pipe.zadd('products:by_rating', {product_id: product_data["rating"]})
pipe.execute()
products[product_id] = product_data
print(f"已创建 {len(products)} 个产品")
# 批量读取产品详情
print("\n=== 批量读取产品详情 ===")
product_ids = list(products.keys())[:10] # 读取前10个产品
start_time = time.time()
pipe = r.pipeline()
# 批量获取产品详情
for product_id in product_ids:
pipe.hgetall(product_id)
product_details = pipe.execute()
batch_read_time = time.time() - start_time
print(f"批量读取 {len(product_ids)} 个产品耗时: {batch_read_time:.4f}秒")
# 处理读取到的数据
print("\n产品详情:")
for i, (product_id, details) in enumerate(zip(product_ids, product_details)):
name = details.get(b'name', b'').decode()
price = details.get(b'price', b'').decode()
category = details.get(b'category', b'').decode()
print(f" {i+1}. {product_id}: {name} - ${price} ({category})")
# 按分类批量统计
print("\n=== 按分类统计 ===")
pipe = r.pipeline()
for category in categories:
pipe.scard(f'category:{category}')
category_counts = pipe.execute()
print("各分类产品数量:")
for category, count in zip(categories, category_counts):
print(f" {category}: {count} 个产品")
batch_read_and_process()
---
b.分布式Pipeline
a.多Redis实例
在多个Redis实例之间分配Pipeline操作。
b.负载均衡
根据键的分布策略将操作分发到不同的Redis实例。
c.分布式示例
---
# 分布式Pipeline示例
class DistributedRedisPipeline:
def __init__(self, redis_instances):
"""
:param redis_instances: Redis实例列表 [(host, port), ...]
"""
self.instances = []
for host, port in redis_instances:
client = redis.Redis(host=host, port=port, db=0)
self.instances.append(client)
self.instance_count = len(self.instances)
def _get_instance_index(self, key):
"""根据键名计算应该使用的Redis实例索引"""
return hash(key) % self.instance_count
def distributed_set(self, key_value_pairs):
"""分布式批量设置键值对"""
# 按实例分组操作
instance_operations = [[] for _ in range(self.instance_count)]
for key, value in key_value_pairs:
instance_idx = self._get_instance_index(key)
instance_operations[instance_idx].append((key, value))
# 为每个实例创建Pipeline并执行
results = []
for i, operations in enumerate(instance_operations):
if operations:
pipe = self.instances[i].pipeline()
for key, value in operations:
pipe.set(key, value)
result = pipe.execute()
results.extend(result)
return results
def distributed_get(self, keys):
"""分布式批量获取键值"""
# 按实例分组操作
instance_operations = [[] for _ in range(self.instance_count)]
key_mapping = {} # 记录键到实例的映射
for key in keys:
instance_idx = self._get_instance_index(key)
instance_operations[instance_idx].append(key)
key_mapping[key] = instance_idx
# 为每个实例创建Pipeline并执行
all_results = {}
for i, operations in enumerate(instance_operations):
if operations:
pipe = self.instances[i].pipeline()
for key in operations:
pipe.get(key)
results = pipe.execute()
# 将结果映射回键
for key, result in zip(operations, results):
all_results[key] = result
# 按原始键顺序返回结果
return [all_results[key] for key in keys]
# 演示分布式Pipeline（实际使用需要多个Redis实例）
def demonstrate_distributed_pipeline():
"""演示分布式Pipeline的使用"""
print("=== 分布式Pipeline演示 ===")
# 注意：这里使用同一个Redis实例的不同数据库来模拟
# 实际使用时应该配置不同的Redis服务器
redis_configs = [
('localhost', 6379), # 实际应该是不同的Redis服务器
]
# 由于只有一个Redis实例，我们使用不同DB来模拟
class MockDistributedPipeline:
def __init__(self):
self.instances = [
redis.Redis(host='localhost', port=6379, db=0),
redis.Redis(host='localhost', port=6379, db=1),
redis.Redis(host='localhost', port=6379, db=2)
]
self.instance_count = len(self.instances)
def _get_instance_index(self, key):
return hash(key) % self.instance_count
def distributed_set(self, key_value_pairs):
instance_operations = [[] for _ in range(self.instance_count)]
for key, value in key_value_pairs:
instance_idx = self._get_instance_index(key)
instance_operations[instance_idx].append((key, value))
results = []
for i, operations in enumerate(instance_operations):
if operations:
pipe = self.instances[i].pipeline()
for key, value in operations:
pipe.set(key, value)
result = pipe.execute()
results.extend(result)
return results
def distributed_get(self, keys):
instance_operations = [[] for _ in range(self.instance_count)]
key_mapping = {}
for key in keys:
instance_idx = self._get_instance_index(key)
instance_operations[instance_idx].append(key)
key_mapping[key] = instance_idx
all_results = {}
for i, operations in enumerate(instance_operations):
if operations:
pipe = self.instances[i].pipeline()
for key in operations:
pipe.get(key)
results = pipe.execute()
for key, result in zip(operations, results):
all_results[key] = result
return [all_results[key] for key in keys]
# 使用模拟的分布式Pipeline
dist_pipeline = MockDistributedPipeline()
# 清空所有模拟实例的数据
for instance in dist_pipeline.instances:
instance.flushdb()
# 批量设置数据
test_data = [(f'user:{i}', f'userdata:{i}') for i in range(1, 21)]
print("分布式批量设置数据...")
start_time = time.time()
set_results = dist_pipeline.distributed_set(test_data)
set_time = time.time() - start_time
print(f"设置 {len(test_data)} 个键值对耗时: {set_time:.4f}秒")
# 批量获取数据
keys = [f'user:{i}' for i in range(1, 21)]
print("\n分布式批量获取数据...")
start_time = time.time()
get_results = dist_pipeline.distributed_get(keys)
get_time = time.time() - start_time
print(f"获取 {len(keys)} 个键值耗时: {get_time:.4f}秒")
# 验证结果
print("\n前5个获取结果:")
for i in range(5):
key = keys[i]
value = get_results[i]
if value:
print(f" {key}: {value.decode()}")
else:
print(f" {key}: None")
demonstrate_distributed_pipeline()
---
04.最佳实践
a.性能优化
a.批处理大小
控制每个Pipeline的命令数量，通常在100-1000个之间。
b.错误处理
实现Pipeline执行的错误重试机制。
c.内存管理
避免在Pipeline中堆积过多命令导致内存占用过高。
d.性能优化示例
---
# Pipeline性能优化示例
class OptimizedRedisPipeline:
def __init__(self, redis_client, batch_size=500):
self.r = redis_client
self.batch_size = batch_size
def optimized_bulk_insert(self, items):
"""
优化的批量插入方法
:param items: [(key, value), ...] 格式的键值对列表
"""
total_items = len(items)
processed = 0
results = []
print(f"开始批量插入 {total_items} 个项目 (批次大小: {self.batch_size})")
start_time = time.time()
# 分批处理
while processed < total_items:
# 计算当前批次
batch_end = min(processed + self.batch_size, total_items)
batch_items = items[processed:batch_end]
try:
# 创建Pipeline
pipe = self.r.pipeline()
# 批量添加命令
for key, value in batch_items:
# 根据值的类型选择合适的Redis命令
if isinstance(value, dict):
pipe.hset(key, mapping=value)
elif isinstance(value, (list, tuple)):
pipe.rpush(key, *value)
else:
pipe.set(key, value)
# 执行当前批次
batch_results = pipe.execute()
results.extend(batch_results)
processed = batch_end
# 进度报告
progress = (processed / total_items) * 100
print(f"进度: {processed}/{total_items} ({progress:.1f}%)")
except Exception as e:
print(f"批次 {processed//self.batch_size + 1} 执行失败: {e}")
# 可以选择重试或跳过失败的批次
break
total_time = time.time() - start_time
print(f"批量插入完成，总耗时: {total_time:.4f}秒")
print(f"平均每秒处理: {total_items/total_time:.0f} 个项目")
return results
def optimized_bulk_read(self, keys):
"""
优化的批量读取方法
:param keys: 要读取的键列表
"""
total_keys = len(keys)
results = {}
print(f"开始批量读取 {total_keys} 个键")
start_time = time.time()
# 分批读取
for i in range(0, total_keys, self.batch_size):
batch_keys = keys[i:i + self.batch_size]
try:
# 创建Pipeline
pipe = self.r.pipeline()
# 批量添加GET命令
for key in batch_keys:
pipe.get(key)
# 执行并获取结果
batch_results = pipe.execute()
# 映射结果
for key, value in zip(batch_keys, batch_results):
results[key] = value
except Exception as e:
print(f"读取批次 {i//self.batch_size + 1} 失败: {e}")
# 对于读取失败，可以记录失败的键
for key in batch_keys:
results[key] = None
total_time = time.time() - start_time
print(f"批量读取完成，总耗时: {total_time:.4f}秒")
print(f"平均每秒读取: {total_keys/total_time:.0f} 个键")
return results
# 演示优化的Pipeline使用
def demonstrate_optimized_pipeline():
"""演示优化的Pipeline使用方法"""
print("=== 优化Pipeline性能演示 ===")
# 创建优化的Pipeline处理器
optimized_pipeline = OptimizedRedisPipeline(r, batch_size=200)
# 准备测试数据
print("\n准备测试数据...")
test_data = []
for i in range(1000):
key = f'optimized:item:{i:04d}'
if i % 3 == 0:
# Hash类型数据
value = {
'id': i,
'name': f'Item {i}',
'category': f'Category {i % 10}',
'price': i * 1.5
}
elif i % 3 == 1:
# List类型数据
value = [f'tag{i}', f'tag{i+100}', f'tag{i+200}']
else:
# String类型数据
value = f'string_value_{i}'
test_data.append((key, value))
# 执行优化的批量插入
print("\n执行优化的批量插入:")
insert_results = optimized_pipeline.optimized_bulk_insert(test_data)
# 执行优化的批量读取
print("\n执行优化的批量读取:")
keys = [item[0] for item in test_data[:500]] # 读取前500个
read_results = optimized_pipeline.optimized_bulk_read(keys)
# 验证读取结果
print(f"\n读取结果验证:")
success_count = sum(1 for v in read_results.values() if v is not None)
print(f"成功读取: {success_count}/{len(keys)} 个键")
# 显示前几个结果示例
print("\n前5个读取结果示例:")
for i, (key, value) in enumerate(list(read_results.items())[:5]):
if value:
if i % 3 == 0:
# Hash数据需要解析
try:
# 这里简化处理，实际应该根据数据类型解析
print(f" {key}: {value[:50]}... (Hash数据)")
except:
print(f" {key}: {value}")
else:
print(f" {key}: {value.decode()[:50]}...")
else:
print(f" {key}: None")
demonstrate_optimized_pipeline()
---
7.4 Lua脚本原子性
01.Lua脚本概述
a.原子性保证
Redis将Lua脚本作为单个原子操作执行，不会被其他命令打断。
b.执行机制
a.脚本加载
使用SCRIPT LOAD或EVAL命令加载和执行Lua脚本。
b.环境隔离
每个脚本在独立的Lua环境中执行，具有全局变量作用域。
c.参数传递
可以通过KEYS和ARGV数组向脚本传递参数。
c.性能优势
a.减少网络往返
复杂逻辑在服务器端执行，减少客户端和服务器之间的通信。
b.原子性保证
避免使用WATCH/UNWATCH的复杂性，直接保证操作的原子性。
d.代码示例
---
# Lua脚本基本示例
import redis
r = redis.Redis(host='localhost', port=6379, db=0)
def basic_lua_script_example():
"""演示Lua脚本的基本使用"""
# 设置初始数据
r.set('counter', 10)
r.set('multiplier', 3)
# 简单的Lua脚本：读取两个键的值，相乘后设置到第三个键
lua_script = """
local value1 = redis.call('GET', KEYS[1])
local value2 = redis.call('GET', KEYS[2])
if value1 and value2 then
local result = tonumber(value1) * tonumber(value2)
redis.call('SET', KEYS[3], result)
return result
else
return nil
end
"""
# 执行Lua脚本
result = r.eval(lua_script, 3, 'counter', 'multiplier', 'result')
print(f"脚本执行结果: {result}")
# 验证结果
result_value = r.get('result')
print(f"Redis中的结果: {result_value}")
basic_lua_script_example()
---
02.Lua脚本语法
a.基本语法规则
a.注释方式
使用--进行单行注释，--[[ ]]进行多行注释。
b.变量声明
支持local变量和全局变量，推荐使用local。
c.数据类型
支持数字、字符串、布尔值、表等基本数据类型。
b.Redis命令调用
a.redis.call函数
同步调用Redis命令，如果命令失败会抛出异常。
b.redis.pcall函数
异步调用Redis命令，错误时返回错误信息而不抛出异常。
c.返回值处理
Redis命令的返回值会自动转换为Lua数据类型。
c.脚本结构示例
---
# Lua脚本语法和结构示例
def lua_script_structure_example():
"""演示Lua脚本的完整结构"""
# 完整的库存扣减脚本
inventory_script = """
-- 库存扣减Lua脚本
-- KEYS[1]: 库存键
-- KEYS[2]: 订单记录键
-- ARGV[1]: 扣减数量
-- ARGV[2]: 用户ID
-- ARGV[3]: 订单ID
local inventory_key = KEYS[1]
local order_key = KEYS[2]
local quantity = tonumber(ARGV[1])
local user_id = ARGV[2]
local order_id = ARGV[3]
-- 检查参数有效性
if not quantity or quantity <= 0 then
return {err = "Invalid quantity"}
end
-- 获取当前库存
local current_stock = redis.call('GET', inventory_key)
if not current_stock then
return {err = "Inventory not found"}
end
current_stock = tonumber(current_stock)
-- 检查库存是否充足
if current_stock < quantity then
return {err = "Insufficient inventory", available = current_stock}
end
-- 执行原子性操作
local new_stock = current_stock - quantity
-- 设置新的库存值
redis.call('SET', inventory_key, new_stock)
-- 记录订单信息
local order_info = string.format(
"user:%s,quantity:%d,time:%d",
user_id, quantity, os.time()
)
redis.call('HSET', order_key, order_id, order_info)
-- 更新统计信息
redis.call('INCRBY', 'stats:total_orders', 1)
redis.call('INCRBY', 'stats:total_quantity', quantity)
return {success = true, new_stock = new_stock, old_stock = current_stock}
"""
# 设置测试数据
r.delete('inventory:product:1001', 'orders:product:1001')
r.set('inventory:product:1001', 100)
print(f"初始库存: {r.get('inventory:product:1001')}")
# 执行库存扣减脚本
result = r.eval(
inventory_script,
2, -- 键的数量
'inventory:product:1001', 'orders:product:1001', -- 键
5, 'user123', 'order001' -- 参数
)
print(f"库存扣减结果: {result}")
# 验证结果
new_stock = r.get('inventory:product:1001')
order_info = r.hget('orders:product:1001', 'order001')
total_orders = r.get('stats:total_orders')
print(f"新库存: {new_stock}")
print(f"订单信息: {order_info}")
print(f"总订单数: {total_orders}")
lua_script_structure_example()
---
03.高级Lua脚本技巧
a.复杂业务逻辑
a.条件判断
使用if-else实现复杂的业务逻辑判断。
b.循环处理
使用for和while循环批量处理数据。
c.错误处理
使用pcall进行异常捕获和处理。
b.高级技巧示例
---
# 高级Lua脚本技巧示例
def advanced_lua_script_techniques():
"""演示高级Lua脚本技巧"""
# 复杂的限流器脚本
rate_limiter_script = """
-- 滑动窗口限流器脚本
-- KEYS[1]: 限流键名
-- ARGV[1]: 时间窗口大小（秒）
-- ARGV[2]: 最大请求数
-- ARGV[3]: 当前时间戳
-- ARGV[4]: 请求ID（可选）
local key = KEYS[1]
local window_size = tonumber(ARGV[1])
local max_requests = tonumber(ARGV[2])
local current_time = tonumber(ARGV[3])
local request_id = ARGV[4] or 'unknown'
-- 计算窗口开始时间
local window_start = current_time - window_size
-- 清理过期记录
redis.call('ZREMRANGEBYSCORE', key, 0, window_start)
-- 获取当前窗口内的请求数
local current_requests = redis.call('ZCARD', key)
-- 检查是否超过限制
if current_requests >= max_requests then
-- 获取所有请求记录用于分析
local requests = redis.call('ZRANGE', key, 0, -1, 'WITHSCORES')
return {
allowed = false,
current_requests = current_requests,
max_requests = max_requests,
reset_time = window_start + window_size,
recent_requests = requests
}
end
-- 添加新请求记录
redis.call('ZADD', key, current_time, request_id)
redis.call('EXPIRE', key, window_size)
return {
allowed = true,
current_requests = current_requests + 1,
max_requests = max_requests,
remaining = max_requests - current_requests - 1
}
"""
# 批量数据聚合脚本
aggregation_script = """
-- 数据聚合脚本
-- KEYS[1]: 数据前缀
-- ARGV[1]: 聚合类型 (sum, avg, min, max, count)
-- ARGV[2]: 键的数量
local prefix = KEYS[1]
local agg_type = ARGV[1]
local key_count = tonumber(ARGV[2])
local results = {}
local total_sum = 0
local total_count = 0
local min_val = math.huge
local max_val = -math.huge
-- 遍历所有键进行聚合
for i = 1, key_count do
local key = prefix .. i
local value = redis.call('GET', key)
if value then
local num_value = tonumber(value)
if num_value then
total_sum = total_sum + num_value
total_count = total_count + 1
if num_value < min_val then
min_val = num_value
end
if num_value > max_val then
max_val = num_value
end
table.insert(results, {key = key, value = num_value})
end
end
end
-- 根据聚合类型返回结果
if agg_type == 'sum' then
return {sum = total_sum, count = total_count}
elseif agg_type == 'avg' then
return {avg = total_sum / total_count, count = total_count}
elseif agg_type == 'min' then
return {min = min_val, count = total_count}
elseif agg_type == 'max' then
return {max = max_val, count = total_count}
elseif agg_type == 'count' then
return {count = total_count}
else
return {
sum = total_sum,
avg = total_sum / total_count,
min = min_val,
max = max_val,
count = total_count,
details = results
}
end
"""
print("=== 高级Lua脚本技巧演示 ===")
# 测试限流器
print("\n1. 滑动窗口限流器:")
limiter_key = 'rate_limiter:test'
r.delete(limiter_key)
import time
current_time = int(time.time())
# 测试在限制内的请求
for i in range(3):
result = r.eval(rate_limiter_script, 1, limiter_key, 10, 5, current_time + i, f'req{i+1}')
print(f"请求 {i+1}: {result}")
# 测试超出限制的请求
result = r.eval(rate_limiter_script, 1, limiter_key, 10, 5, current_time + 3, 'req_exceed')
print(f"超出限制的请求: {result}")
# 测试数据聚合
print("\n2. 数据聚合:")
# 准备测试数据
for i in range(1, 6):
key = f'data:agg:{i}'
value = i * 10
r.set(key, value)
print(f"设置 {key} = {value}")
# 执行聚合
agg_result = r.eval(aggregation_script, 1, 'data:agg:', 'all', 5)
print(f"聚合结果: {agg_result}")
# 单独计算平均值
avg_result = r.eval(aggregation_script, 1, 'data:agg:', 'avg', 5)
print(f"平均值: {avg_result}")
advanced_lua_script_techniques()
---
04.脚本管理和优化
a.脚本缓存
a. SCRIPT LOAD
预先加载脚本到Redis服务器，获取脚本哈希值供后续使用。
b. SCRIPT EXISTS
检查脚本是否已加载到服务器缓存中。
c. SCRIPT FLUSH
清空所有已加载的Lua脚本缓存。
b.脚本优化
a.减少复杂度
简化脚本逻辑，避免不必要的循环和计算。
b.使用局部变量
优先使用local变量提高性能。
c.批量操作
在脚本内进行批量操作减少网络开销。
c.管理和优化示例
---
# 脚本管理和优化示例
class RedisLuaScriptManager:
def __init__(self, redis_client):
self.r = redis_client
self.script_cache = {}
def load_script(self, script_name, script_content):
"""加载并缓存Lua脚本"""
try:
sha = self.r.script_load(script_content)
self.script_cache[script_name] = {
'sha': sha,
'content': script_content,
'load_time': time.time()
}
print(f"脚本 '{script_name}' 加载成功，SHA: {sha}")
return sha
except Exception as e:
print(f"脚本 '{script_name}' 加载失败: {e}")
return None
def execute_cached_script(self, script_name, keys, args):
"""执行已缓存的脚本"""
if script_name not in self.script_cache:
raise ValueError(f"脚本 '{script_name}' 未加载")
sha = self.script_cache[script_name]['sha']
return self.r.evalsha(sha, len(keys), *keys, *args)
def script_exists(self, script_name):
"""检查脚本是否存在"""
if script_name not in self.script_cache:
return False
sha = self.script_cache[script_name]['sha']
try:
result = self.r.script_exists(sha)
return result[0] # script_exists返回列表
except:
return False
def optimize_script_execution(self, script_name, keys, args, batch_size=100):
"""优化的批量脚本执行"""
if script_name not in self.script_cache:
raise ValueError(f"脚本 '{script_name}' 未加载")
sha = self.script_cache[script_name]['sha']
# 如果参数数量小于批量大小，直接执行
if len(args) <= batch_size:
return self.r.evalsha(sha, len(keys), *keys, *args)
# 分批执行
results = []
for i in range(0, len(args), batch_size):
batch_args = args[i:i + batch_size]
batch_keys = keys + [str(j) for j in range(len(keys), len(keys) + len(batch_args))]
try:
batch_result = self.r.evalsha(sha, len(batch_keys), *batch_keys, *batch_args)
if isinstance(batch_result, list):
results.extend(batch_result)
else:
results.append(batch_result)
except Exception as e:
print(f"批量执行失败 (批次 {i//batch_size + 1}): {e}")
results.append(None)
return results
def demonstrate_script_management():
"""演示脚本管理和优化"""
print("=== Lua脚本管理演示 ===")
script_manager = RedisLuaScriptManager(r)
# 定义几个常用脚本
scripts = {
'counter_increment': '''
local current = redis.call('GET', KEYS[1]) or 0
local increment = tonumber(ARGV[1]) or 1
local new_value = current + increment
redis.call('SET', KEYS[1], new_value)
return new_value
''',
'smart_cache': '''
local key = KEYS[1]
local ttl = ARGV[1]
local default_value = ARGV[2]
local value = redis.call('GET', key)
if value then
-- 存在缓存，更新过期时间
redis.call('EXPIRE', key, ttl)
return {cached = true, value = value}
else
-- 不存在缓存，设置默认值
redis.call('SET', key, default_value)
redis.call('EXPIRE', key, ttl)
return {cached = false, value = default_value}
end
''',
'batch_delete': '''
local deleted_count = 0
for i = 1, #ARGV do
local key = ARGV[i]
local result = redis.call('DEL', key)
deleted_count = deleted_count + result
end
return {deleted = deleted_count, total = #ARGV}
'''
}
# 加载所有脚本
print("\n加载脚本:")
loaded_scripts = {}
for name, content in scripts.items():
sha = script_manager.load_script(name, content)
if sha:
loaded_scripts[name] = sha
# 验证脚本加载状态
print("\n验证脚本加载状态:")
for name in loaded_scripts.keys():
exists = script_manager.script_exists(name)
print(f" {name}: {'已加载' if exists else '未加载'}")
# 使用已加载的脚本
print("\n使用已加载的脚本:")
# 测试计数器脚本
r.delete('test:counter')
result = script_manager.execute_cached_script('counter_increment', ['test:counter'], [5])
print(f"计数器结果: {result}")
# 测试智能缓存脚本
r.delete('test:cache')
result = script_manager.execute_cached_script('smart_cache', ['test:cache'], [300, 'default'])
print(f"缓存设置: {result}")
# 再次读取缓存
result = script_manager.execute_cached_script('smart_cache', ['test:cache'], [300, 'default'])
print(f"缓存读取: {result}")
# 测试批量删除
test_keys = [f'test:del:{i}' for i in range(1, 6)]
for key in test_keys:
r.set(key, f'value_{key}')
result = script_manager.execute_cached_script('batch_delete', [], test_keys)
print(f"批量删除结果: {result}")
demonstrate_script_management()
---
7.5 事务失败处理
01.失败类型分析
a.语法错误
a.命令格式错误
命令参数格式不正确或缺少必要参数。
b.无效命令
使用了Redis不支持的命令。
c.参数类型错误
传递了错误的数据类型给Redis命令。
b.运行时错误
a.键不存在
尝试操作不存在的键导致操作失败。
b.数据类型错误
对错误的Redis数据类型执行不兼容的操作。
c.权限错误
没有足够权限执行某些操作。
c.代码示例
---
# 事务失败类型示例
import redis
import time
r = redis.Redis(host='localhost', port=6379, db=0)
def analyze_transaction_failures():
"""分析不同类型的事务失败"""
print("=== 事务失败类型分析 ===")
# 1. 语法错误示例
print("\n1. 语法错误:")
try:
pipe = r.pipeline(transaction=True)
pipe.set('key1', 'value1')
# 故意使用错误的命令
pipe.execute_command('INVALID_COMMAND', 'arg')
pipe.set('key2', 'value2')
results = pipe.execute()
print(f"意外成功: {results}")
except Exception as e:
print(f"预期的语法错误: {e}")
# 2. 数据类型错误示例
print("\n2. 数据类型错误:")
r.delete('test:list')
r.lpush('test:list', 'item1', 'item2') # 创建列表
try:
pipe = r.pipeline(transaction=True)
# 对列表执行字符串操作会失败
pipe.append('test:list', 'string_append')
pipe.incr('test:list') # 对列表执行INCR操作
results = pipe.execute()
print(f"操作结果: {results}")
except Exception as e:
print(f"数据类型错误: {e}")
# 3. 键不存在错误
print("\n3. 键不存在的处理:")
try:
pipe = r.pipeline(transaction=True)
pipe.incr('nonexistent:counter') # 不存在的键
pipe.hget('nonexistent:hash', 'field') # 不存在的哈希
results = pipe.execute()
print(f"处理不存在键的结果: {results}")
except Exception as e:
print(f"键不存在错误: {e}")
analyze_transaction_failures()
---
02.错误处理策略
a.预防性检查
a.键存在性检查
在事务开始前检查必要的键是否存在。
b.数据类型验证
确认键的数据类型与要执行的操作兼容。
c.参数验证
在入队前验证参数的有效性。
b.容错机制
a.降级处理
在关键操作失败时提供替代方案。
b.重试机制
对于临时性错误实现智能重试。
c.回滚策略
部分失败时的数据恢复机制。
c.错误处理示例
---
# 事务错误处理策略示例
class RobustTransactionManager:
def __init__(self, redis_client):
self.r = redis_client
self.max_retries = 3
self.retry_delay = 0.1
def safe_account_transfer(self, from_account, to_account, amount):
"""安全的转账事务，包含完整的错误处理"""
for attempt in range(self.max_retries):
try:
# 预检查
if not self._validate_transfer_data(from_account, to_account, amount):
return False, "预检查失败"
# 执行事务
success, result = self._execute_transfer_transaction(
from_account, to_account, amount
)
if success:
return True, result
elif attempt < self.max_retries - 1:
time.sleep(self.retry_delay * (2 ** attempt))
continue
else:
return False, "达到最大重试次数"
except redis.RedisError as e:
if attempt < self.max_retries - 1:
time.sleep(self.retry_delay * (2 ** attempt))
continue
else:
return False, f"Redis错误: {e}"
except Exception as e:
return False, f"未知错误: {e}"
return False, "未知失败"
def _validate_transfer_data(self, from_account, to_account, amount):
"""预检查转账数据的有效性"""
try:
# 检查参数类型
if not isinstance(amount, (int, float)) or amount <= 0:
return False
# 检查账户键是否存在
if not self.r.exists(from_account):
return False
if not self.r.exists(to_account):
# 如果目标账户不存在，先创建
self.r.set(to_account, 0)
# 检查源账户余额
balance = int(self.r.get(from_account) or 0)
if balance < amount:
return False
return True
except:
return False
def _execute_transfer_transaction(self, from_account, to_account, amount):
"""执行转账事务"""
try:
pipe = self.r.pipeline(transaction=True)
# 读取源账户余额
source_balance = pipe.get(from_account)
# 检查余额是否足够
# 注意：这里需要特殊处理，因为我们在事务中不能直接比较数值
# 更好的方法是使用Lua脚本或乐观锁
pipe.decrby(from_account, int(amount))
pipe.incrby(to_account, int(amount))
# 记录转账日志
timestamp = int(time.time())
log_entry = f"{from_account}->{to_account}:{amount}:{timestamp}"
pipe.lpush('transaction:log', log_entry)
# 执行事务
results = pipe.execute()
# 验证转账是否成功
if results[0] is not None: # DECRBY成功
return True, f"转账成功，新余额: {results[0]}"
else:
return False, "余额不足"
except Exception as e:
return False, f"事务执行失败: {e}"
def batch_update_with_rollback(self, updates):
"""批量更新，支持部分失败时的回滚"""
original_values = {}
updated_keys = []
try:
# 记录原始值
for key, value in updates.items():
if self.r.exists(key):
original_values[key] = self.r.get(key)
else:
original_values[key] = None
# 执行批量更新
pipe = self.r.pipeline(transaction=True)
for key, value in updates.items():
if isinstance(value, dict):
pipe.hset(key, mapping=value)
else:
pipe.set(key, value)
updated_keys.append(key)
results = pipe.execute()
return True, {"updated": len(updated_keys), "results": results}
except Exception as e:
# 回滚操作
try:
rollback_pipe = self.r.pipeline()
for key in updated_keys:
original_value = original_values[key]
if original_value is not None:
rollback_pipe.set(key, original_value)
else:
rollback_pipe.delete(key)
rollback_pipe.execute()
return False, f"更新失败，已回滚: {e}"
except Exception as rollback_error:
return False, f"更新失败且回滚也失败: {e}, {rollback_error}"
# 测试错误处理策略
def test_error_handling_strategies():
"""测试事务错误处理策略"""
print("=== 错误处理策略测试 ===")
manager = RobustTransactionManager(r)
# 设置测试账户
r.set('account:alice', 1000)
r.set('account:bob', 500)
# 测试1: 正常转账
print("\n1. 正常转账测试:")
success, message = manager.safe_account_transfer('account:alice', 'account:bob', 200)
print(f"转账结果: {success}, {message}")
print(f"Alice余额: {r.get('account:alice')}, Bob余额: {r.get('account:bob')}")
# 测试2: 余额不足
print("\n2. 余额不足测试:")
success, message = manager.safe_account_transfer('account:alice', 'account:bob', 1000)
print(f"转账结果: {success}, {message}")
# 测试3: 账户不存在
print("\n3. 目标账户不存在测试:")
r.delete('account:charlie')
success, message = manager.safe_account_transfer('account:alice', 'account:charlie', 100)
print(f"转账结果: {success}, {message}")
print(f"Charlie余额: {r.get('account:charlie')}")
# 测试4: 批量更新回滚
print("\n4. 批量更新回滚测试:")
test_updates = {
'test:user:1': {'name': 'Alice', 'age': 25},
'test:user:2': {'name': 'Bob', 'age': 30}
}
# 故意设置一个会导致错误的情况
r.set('test:user:1', 'string_data') # 错误的数据类型
success, message = manager.batch_update_with_rollback(test_updates)
print(f"批量更新结果: {success}, {message}")
print(f"test:user:1的值: {r.get('test:user:1')}")
test_error_handling_strategies()
---
03.监控与日志
a.事务监控
a.执行统计
监控事务的成功率、失败率和平均执行时间。
b.性能指标
跟踪事务的吞吐量、延迟和资源使用情况。
c.错误分析
统计不同类型错误的发生频率。
b.日志记录
a.事务日志
记录每个事务的详细信息，包括参与的关键和操作。
b.错误日志
详细记录失败事务的错误信息和上下文。
c.审计日志
记录敏感操作和重要业务流程。
c.监控日志示例
---
# 事务监控和日志示例
import logging
import json
from datetime import datetime
class TransactionMonitor:
def __init__(self, redis_client):
self.r = redis_client
self.logger = self._setup_logger()
self.stats = {
'total_transactions': 0,
'successful_transactions': 0,
'failed_transactions': 0,
'error_types': {},
'execution_times': []
}
def _setup_logger(self):
"""设置日志记录器"""
logger = logging.getLogger('transaction_monitor')
logger.setLevel(logging.INFO)
# 文件处理器
file_handler = logging.FileHandler('redis_transactions.log')
file_handler.setLevel(logging.INFO)
# 控制台处理器
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.WARNING)
# 格式化器
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
logger.addHandler(file_handler)
logger.addHandler(console_handler)
return logger
def monitor_transaction(self, operation_name, keys, operation_func):
"""监控事务执行"""
start_time = time.time()
self.stats['total_transactions'] += 1
# 记录事务开始
transaction_id = f"txn_{int(time.time() * 1000)}"
self.logger.info(f"Transaction {transaction_id} started: {operation_name}")
try:
# 执行事务
result = operation_func()
execution_time = time.time() - start_time
# 更新统计信息
self.stats['successful_transactions'] += 1
self.stats['execution_times'].append(execution_time)
# 记录成功
self.logger.info(
f"Transaction {transaction_id} succeeded: {operation_name} "
f"in {execution_time:.4f}s"
)
# 记录审计信息
audit_data = {
'transaction_id': transaction_id,
'operation': operation_name,
'keys': keys,
'status': 'success',
'execution_time': execution_time,
'timestamp': datetime.now().isoformat()
}
self._log_audit(audit_data)
return True, result
except Exception as e:
execution_time = time.time() - start_time
error_type = type(e).__name__
# 更新错误统计
self.stats['failed_transactions'] += 1
self.stats['error_types'][error_type] = \
self.stats['error_types'].get(error_type, 0) + 1
# 记录错误
error_msg = f"Transaction {transaction_id} failed: {operation_name} - {error_type}: {e}"
self.logger.error(error_msg)
# 记录错误审计信息
audit_data = {
'transaction_id': transaction_id,
'operation': operation_name,
'keys': keys,
'status': 'failed',
'error_type': error_type,
'error_message': str(e),
'execution_time': execution_time,
'timestamp': datetime.now().isoformat()
}
self._log_audit(audit_data)
return False, str(e)
def _log_audit(self, audit_data):
"""记录审计日志"""
try:
# 使用Redis列表存储审计日志
audit_key = 'audit:transactions'
self.r.lpush(audit_key, json.dumps(audit_data))
self.r.ltrim(audit_key, 0, 9999) # 保留最近10000条记录
except Exception as e:
self.logger.error(f"Failed to log audit data: {e}")
def get_statistics(self):
"""获取监控统计信息"""
if self.stats['execution_times']:
avg_time = sum(self.stats['execution_times']) / len(self.stats['execution_times'])
max_time = max(self.stats['execution_times'])
min_time = min(self.stats['execution_times'])
else:
avg_time = max_time = min_time = 0
total = self.stats['total_transactions']
success_rate = (self.stats['successful_transactions'] / total * 100) if total > 0 else 0
return {
'total_transactions': total,
'successful_transactions': self.stats['successful_transactions'],
'failed_transactions': self.stats['failed_transactions'],
'success_rate': f"{success_rate:.2f}%",
'average_execution_time': f"{avg_time:.4f}s",
'max_execution_time': f"{max_time:.4f}s",
'min_execution_time': f"{min_time:.4f}s",
'error_types': self.stats['error_types']
}
def get_recent_failures(self, limit=10):
"""获取最近的失败记录"""
try:
# 从审计日志中获取失败的记录
audit_logs = self.r.lrange('audit:transactions', 0, -1)
failures = []
for log_str in audit_logs:
try:
log_data = json.loads(log_str)
if log_data.get('status') == 'failed':
failures.append(log_data)
if len(failures) >= limit:
break
except:
continue
return failures
except Exception as e:
self.logger.error(f"Failed to get recent failures: {e}")
return []
# 测试监控功能
def test_transaction_monitoring():
"""测试事务监控功能"""
print("=== 事务监控测试 ===")
monitor = TransactionMonitor(r)
# 测试1: 成功的事务
def successful_transaction():
pipe = r.pipeline(transaction=True)
pipe.set('test:mon1', 'value1')
pipe.set('test:mon2', 'value2')
return pipe.execute()
success, result = monitor.monitor_transaction(
'test_success', ['test:mon1', 'test:mon2'], successful_transaction
)
print(f"成功事务监控: {success}")
# 测试2: 失败的事务
def failing_transaction():
pipe = r.pipeline(transaction=True)
pipe.set('test:mon3', 'value3')
pipe.execute_command('INVALID_COMMAND') # 故意失败
pipe.set('test:mon4', 'value4')
return pipe.execute()
success, error = monitor.monitor_transaction(
'test_failure', ['test:mon3', 'test:mon4'], failing_transaction
)
print(f"失败事务监控: {success}, 错误: {error}")
# 显示统计信息
print("\n=== 监控统计 ===")
stats = monitor.get_statistics()
for key, value in stats.items():
print(f"{key}: {value}")
# 显示最近的失败记录
print("\n=== 最近的失败记录 ===")
failures = monitor.get_recent_failures(3)
for i, failure in enumerate(failures, 1):
print(f"{i}. {failure.get('timestamp')}: {failure.get('error_type')} - {failure.get('error_message')}")
test_transaction_monitoring()
---
8. 文件系统事务
8.1 原子性文件操作
01.文件原子操作原理
a.原子性定义
文件操作要么完全成功，要么完全失败，不会出现部分完成的状态。
b.传统文件系统问题
a.写入中断
程序崩溃或系统故障可能导致文件写入不完整。
b.并发访问
多个进程同时写入同一文件可能导致数据损坏。
c.缓存问题
操作系统缓存可能导致数据未真正写入磁盘。
c.解决方案
a.临时文件模式
先写入临时文件，完成后重命名为目标文件。
b.写前日志
记录操作日志，确保可以恢复或回滚。
c.锁机制
使用文件锁防止并发冲突。
d.代码示例
---
# 文件原子操作基础示例
import os
import tempfile
import shutil
import json
class AtomicFileWriter:
def __init__(self, filepath, encoding='utf-8'):
self.filepath = filepath
self.encoding = encoding
self.temp_file = None
def __enter__(self):
# 创建临时文件在目标文件的同一目录
dirname = os.path.dirname(self.filepath)
if not dirname:
dirname = '.'
# 生成唯一的临时文件名
fd, self.temp_file = tempfile.mkstemp(
dir=dirname,
prefix='.tmp_' + os.path.basename(self.filepath) + '_',
suffix='.tmp'
)
os.close(fd) # 关闭文件描述符，我们使用文件对象
return open(self.temp_file, 'w', encoding=self.encoding)
def __exit__(self, exc_type, exc_val, exc_tb):
# 关闭临时文件
if os.path.exists(self.temp_file):
try:
with open(self.temp_file, 'r', encoding=self.encoding) as f:
# 检查文件是否有内容
if f.read(1): # 至少有一个字符
# 如果没有异常发生，将临时文件重命名为目标文件
if exc_type is None:
if os.path.exists(self.filepath):
# 在Windows上，需要先删除目标文件
os.remove(self.filepath)
os.rename(self.temp_file, self.filepath)
else:
# 如果有异常，删除临时文件
os.remove(self.temp_file)
else:
# 临时文件为空，删除它
os.remove(self.temp_file)
except Exception as e:
print(f"Error in atomic write: {e}")
# 尝试清理临时文件
if os.path.exists(self.temp_file):
os.remove(self.temp_file)
def atomic_write_json(self, data):
"""原子性写入JSON数据"""
with self as f:
json.dump(data, f, indent=2, ensure_ascii=False)
def atomic_write_text(self, text):
"""原子性写入文本数据"""
with self as f:
f.write(text)
def atomic_append(self, text):
"""原子性追加文本（创建新文件）"""
# 先读取现有内容
existing_content = ""
if os.path.exists(self.filepath):
with open(self.filepath, 'r', encoding=self.encoding) as f:
existing_content = f.read()
# 写入新内容
with self as f:
f.write(existing_content + text)
# 基本使用示例
def demo_basic_atomic_operations():
print("=== 基本原子操作演示 ===")
# 1. 原子性写入文本
writer = AtomicFileWriter('test_atomic.txt')
writer.atomic_write_text("这是第一行\n这是第二行\n")
print("文本写入完成")
# 验证文件内容
with open('test_atomic.txt', 'r', encoding='utf-8') as f:
content = f.read()
print(f"文件内容: {repr(content)}")
# 2. 原子性追加
writer.atomic_append("\n这是追加的第三行\n")
print("文本追加完成")
# 3. 原子性写入JSON
data = {
"name": "Alice",
"age": 25,
"skills": ["Python", "Redis", "Database"]
}
json_writer = AtomicFileWriter('user_data.json')
json_writer.atomic_write_json(data)
print("JSON写入完成")
# 验证JSON内容
with open('user_data.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
print(f"JSON数据: {json_data}")
demo_basic_atomic_operations()
---
02.并发文件操作
a.并发访问问题
a.竞态条件
多个进程同时修改文件可能导致数据不一致。
b.部分写入
一个进程写入被另一个进程中断，导致文件损坏。
c.读取不一致
读取文件时文件正在被修改，可能读到不完整数据。
b.锁机制
a.文件锁
使用操作系统级别的文件锁机制。
b.目录锁
锁定整个目录来管理其中的文件操作。
c.进程锁
使用PID文件实现进程级别的锁定。
c.并发处理示例
---
# 并发文件操作示例
import fcntl # Unix文件锁
import threading
import time
from contextlib import contextmanager
class ConcurrentFileWriter:
def __init__(self, filepath):
self.filepath = filepath
self.lock_file = filepath + '.lock'
@contextmanager
def acquire_lock(self):
"""获取文件锁"""
max_retries = 10
retry_delay = 0.1
for i in range(max_retries):
try:
# 尝试创建锁文件（原子操作）
fd = os.open(self.lock_file, os.O_CREAT | os.O_EXCL | os.O_WRONLY)
os.close(fd)
yield
return
except OSError:
if i < max_retries - 1:
time.sleep(retry_delay * (2 ** i))
else:
raise Exception(f"无法获取文件锁: {self.filepath}")
def release_lock(self):
"""释放文件锁"""
try:
os.remove(self.lock_file)
except FileNotFoundError:
pass # 锁文件可能已被其他进程删除
def safe_write(self, content):
"""安全的并发写入"""
with self.acquire_lock():
# 写入到临时文件
temp_file = self.filepath + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
f.write(content)
# 原子性重命名
if os.path.exists(self.filepath):
os.remove(self.filepath)
os.rename(temp_file, self.filepath)
finally:
self.release_lock()
def safe_read(self):
"""安全的并发读取"""
with self.acquire_lock():
try:
with open(self.filepath, 'r', encoding='utf-8') as f:
return f.read()
except FileNotFoundError:
return None
def safe_append(self, content):
"""安全的并发追加"""
with self.acquire_lock():
try:
# 读取现有内容
existing_content = ""
if os.path.exists(self.filepath):
with open(self.filepath, 'r', encoding='utf-8') as f:
existing_content = f.read()
# 写入完整内容
with open(self.filepath, 'w', encoding='utf-8') as f:
f.write(existing_content + content)
except Exception as e:
print(f"追加操作失败: {e}")
finally:
self.release_lock()
# 并发测试示例
def test_concurrent_operations():
"""测试并发文件操作"""
print("=== 并发操作测试 ===")
writer = ConcurrentFileWriter('shared_counter.txt')
# 初始化文件
writer.safe_write("0")
def worker_worker(worker_id, operations):
"""工作线程函数"""
for i in range(operations):
try:
# 读取当前计数
current = writer.safe_read()
if current:
counter = int(current.strip())
else:
counter = 0
# 模拟一些处理时间
time.sleep(0.01)
# 写入新计数
new_counter = counter + 1
writer.safe_write(str(new_counter))
print(f"Worker {worker_id}: 操作 {i+1}, 计数器 = {new_counter}")
except Exception as e:
print(f"Worker {worker_id} 出错: {e}")
time.sleep(0.001)
# 启动多个工作线程
threads = []
for i in range(3):
thread = threading.Thread(target=worker_worker, args=(i+1, 10))
threads.append(thread)
thread.start()
# 等待所有线程完成
for thread in threads:
thread.join()
# 验证最终结果
final_count = writer.safe_read()
print(f"\n最终计数器值: {final_count}")
test_concurrent_operations()
---
03.数据完整性保障
a.校验机制
a.校验和
计算文件的校验和确保数据完整性。
b.版本控制
维护文件版本号用于检测修改冲突。
c.备份策略
在修改前创建文件备份。
b.恢复机制
a.自动恢复
检测到损坏时自动从备份恢复。
b.手动恢复
提供工具进行手动数据恢复。
c.事务日志
使用日志记录所有操作，支持回滚。
c.完整性保障示例
---
# 文件完整性保障示例
import hashlib
import shutil
import glob
from datetime import datetime
class FileIntegrityManager:
def __init__(self, filepath, backup_dir='backups'):
self.filepath = filepath
self.backup_dir = backup_dir
self.checksum_file = filepath + '.checksum'
self.version_file = filepath + '.version'
# 确保备份目录存在
os.makedirs(backup_dir, exist_ok=True)
def _calculate_checksum(self, filepath):
"""计算文件的MD5校验和"""
hash_md5 = hashlib.md5()
with open(filepath, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
hash_md5.update(chunk)
return hash_md5.hexdigest()
def _get_version(self):
"""获取当前文件版本"""
try:
with open(self.version_file, 'r') as f:
return int(f.read().strip())
except (FileNotFoundError, ValueError):
return 0
def _increment_version(self):
"""递增版本号"""
version = self._get_version()
new_version = version + 1
with open(self.version_file, 'w') as f:
f.write(str(new_version))
return new_version
def create_backup(self):
"""创建文件备份"""
if not os.path.exists(self.filepath):
raise FileNotFoundError(f"源文件不存在: {self.filepath}")
version = self._get_version()
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S')
backup_name = f"v{version}_{timestamp}_{os.path.basename(self.filepath)}"
backup_path = os.path.join(self.backup_dir, backup_name)
shutil.copy2(self.filepath, backup_path)
return backup_path
def verify_integrity(self):
"""验证文件完整性"""
if not os.path.exists(self.filepath):
return False, "文件不存在"
# 检查校验和
current_checksum = self._calculate_checksum(self.filepath)
if os.path.exists(self.checksum_file):
with open(self.checksum_file, 'r') as f:
stored_checksum = f.read().strip()
if current_checksum != stored_checksum:
return False, "校验和不匹配，文件可能已损坏"
return True, "文件完整性验证通过"
def atomic_write_with_integrity(self, content):
"""带完整性保障的原子性写入"""
try:
# 1. 创建备份
backup_path = self.create_backup()
# 2. 写入新内容到临时文件
temp_file = self.filepath + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
f.write(content)
# 3. 计算校验和
checksum = self._calculate_checksum(temp_file)
with open(temp_file + '.checksum', 'w') as f:
f.write(checksum)
# 4. 递增版本
new_version = self._increment_version()
# 5. 原子性替换
if os.path.exists(self.filepath):
os.remove(self.filepath)
os.rename(temp_file, self.filepath)
# 6. 更新校验和文件
if os.path.exists(self.checksum_file):
os.remove(self.checksum_file)
os.rename(temp_file + '.checksum', self.checksum_file)
print(f"文件已更新，版本: {new_version}")
return True, f"更新成功，版本: {new_version}"
except Exception as e:
# 清理临时文件
temp_file = self.filepath + '.tmp'
if os.path.exists(temp_file):
os.remove(temp_file)
return False, f"写入失败: {e}"
def restore_from_backup(self, backup_path=None):
"""从备份恢复文件"""
try:
if backup_path is None:
# 找到最新的备份
backups = glob.glob(os.path.join(self.backup_dir, '*_*'))
if not backups:
return False, "没有可用的备份文件"
backup_path = max(backups, key=os.path.getctime)
if not os.path.exists(backup_path):
return False, f"备份文件不存在: {backup_path}"
# 复制备份文件
if os.path.exists(self.filepath):
os.remove(self.filepath)
shutil.copy2(backup_path, self.filepath)
# 重新计算校验和
checksum = self._calculate_checksum(self.filepath)
with open(self.checksum_file, 'w') as f:
f.write(checksum)
return True, f"已从备份恢复: {backup_path}"
except Exception as e:
return False, f"恢复失败: {e}"
def get_file_info(self):
"""获取文件信息"""
if not os.path.exists(self.filepath):
return None
info = {
'filepath': self.filepath,
'size': os.path.getsize(self.filepath),
'mtime': datetime.fromtimestamp(os.path.getmtime(self.filepath)),
'version': self._get_version(),
'has_checksum': os.path.exists(self.checksum_file)
}
if info['has_checksum']:
with open(self.checksum_file, 'r') as f:
info['checksum'] = f.read().strip()
return info
# 测试完整性保障
def test_integrity_protection():
"""测试文件完整性保障"""
print("=== 文件完整性保障测试 ===")
manager = FileIntegrityManager('important_data.txt')
# 1. 写入初始数据
initial_data = {
"name": "重要系统配置",
"version": "1.0.0",
"settings": {
"timeout": 30,
"retry_count": 3,
"debug_mode": False
}
}
import json
json_str = json.dumps(initial_data, indent=2, ensure_ascii=False)
success, message = manager.atomic_write_with_integrity(json_str)
print(f"初始写入: {success}, {message}")
# 2. 验证完整性
is_valid, message = manager.verify_integrity()
print(f"完整性验证: {is_valid}, {message}")
# 3. 显示文件信息
info = manager.get_file_info()
print(f"\n文件信息:")
for key, value in info.items():
print(f" {key}: {value}")
# 4. 模拟文件损坏（用错误内容替换）
print("\n模拟文件损坏...")
with open('important_data.txt', 'w') as f:
f.write("损坏的数据内容")
# 5. 再次验证完整性
is_valid, message = manager.verify_integrity()
print(f"损坏后验证: {is_valid}, {message}")
# 6. 从备份恢复
print("\n从备份恢复...")
success, message = manager.restore_from_backup()
print(f"恢复结果: {success}, {message}")
# 7. 验证恢复后的完整性
is_valid, message = manager.verify_integrity()
print(f"恢复后验证: {is_valid}, {message}")
test_integrity_protection()
---
8.2 临时文件与重命名
01.临时文件机制原理
a.原子性保证
重命名操作在大多数文件系统中是原子的，要么完全成功，要么完全失败。
b.写入过程
a.创建临时文件
在目标文件的同一目录下创建临时文件。
b.写入数据
将完整数据写入临时文件。
c.原子重命名
将临时文件重命名为目标文件，替换原文件。
c.优势分析
a.避免部分写入
原子重命名确保文件要么是完整的旧版本，要么是完整的新版本。
b.并发安全
多个进程可以安全地并发写入不同的临时文件。
c.崩溃恢复
系统崩溃时最多丢失一个临时文件，不会损坏目标文件。
d.基础实现示例
---
# 临时文件与原子重命名基础示例
import os
import tempfile
import shutil
import hashlib
from pathlib import Path
class AtomicRenameManager:
def __init__(self, target_dir=None):
self.target_dir = target_dir or os.getcwd()
self.temp_files = set()
def create_temp_file(self, target_path, prefix='.tmp_', suffix='.tmp'):
"""在目标文件附近创建临时文件"""
# 确保目标目录存在
target_dir = os.path.dirname(target_path) or '.'
os.makedirs(target_dir, exist_ok=True)
# 生成唯一的临时文件名
target_name = os.path.basename(target_path)
fd, temp_path = tempfile.mkstemp(
dir=target_dir,
prefix=prefix + target_name + '_',
suffix=suffix
)
os.close(fd) # 关闭文件描述符，使用文件对象操作
self.temp_files.add(temp_path)
return temp_path
def atomic_write_with_rename(self, target_path, data, encoding='utf-8'):
"""使用临时文件和重命名实现原子写入"""
temp_path = None
try:
# 创建临时文件
temp_path = self.create_temp_file(target_path)
# 写入数据到临时文件
with open(temp_path, 'w', encoding=encoding) as f:
if isinstance(data, str):
f.write(data)
elif hasattr(data, 'read'):
shutil.copyfileobj(data, f)
else:
f.write(str(data))
# 确保数据写入磁盘
f.flush()
os.fsync(f.fileno())
# 原子重命名
if os.path.exists(target_path):
# Windows需要先删除目标文件
backup_path = target_path + '.bak'
if os.path.exists(backup_path):
os.remove(backup_path)
shutil.move(target_path, backup_path)
shutil.move(temp_path, target_path)
# 清理临时文件记录
self.temp_files.discard(temp_path)
temp_path = None
return True, f"成功写入 {target_path}"
except Exception as e:
# 清理临时文件
if temp_path and os.path.exists(temp_path):
try:
os.remove(temp_path)
except:
pass
return False, f"写入失败: {e}"
def atomic_update_json(self, target_path, data_dict):
"""原子性更新JSON文件"""
import json
# 序列化数据
json_str = json.dumps(data_dict, indent=2, ensure_ascii=False)
return self.atomic_write_with_rename(target_path, json_str, encoding='utf-8')
def atomic_append_with_rename(self, target_path, append_data):
"""原子性追加数据（创建新文件）"""
try:
# 读取现有内容
existing_content = ""
if os.path.exists(target_path):
with open(target_path, 'r', encoding='utf-8') as f:
existing_content = f.read()
# 合并新内容
if isinstance(append_data, str):
new_content = existing_content + append_data
else:
new_content = existing_content + str(append_data)
# 原子写入完整内容
return self.atomic_write_with_rename(target_path, new_content)
except Exception as e:
return False, f"追加失败: {e}"
def cleanup_temp_files(self):
"""清理所有临时文件"""
removed_count = 0
for temp_path in list(self.temp_files):
try:
if os.path.exists(temp_path):
os.remove(temp_path)
removed_count += 1
self.temp_files.discard(temp_path)
except Exception as e:
print(f"清理临时文件失败 {temp_path}: {e}")
return removed_count
def demo_basic_atomic_rename():
"""演示基础的原子重命名操作"""
print("=== 原子重命名基础演示 ===")
manager = AtomicRenameManager()
# 1. 基本文本写入
print("\n1. 基本文本写入:")
success, message = manager.atomic_write_with_rename('test_atomic.txt', 'Hello, Atomic World!\n')
print(f"结果: {success}, {message}")
# 验证文件内容
with open('test_atomic.txt', 'r') as f:
content = f.read()
print(f"文件内容: {repr(content)}")
# 2. JSON数据写入
print("\n2. JSON数据写入:")
test_data = {
"user_id": 12345,
"name": "Alice",
"settings": {
"theme": "dark",
"notifications": True,
"timeout": 300
}
}
success, message = manager.atomic_update_json('config.json', test_data)
print(f"结果: {success}, {message}")
# 验证JSON内容
import json
with open('config.json', 'r', encoding='utf-8') as f:
json_data = json.load(f)
print(f"JSON数据: {json_data}")
# 3. 原子追加
print("\n3. 原子追加:")
success, message = manager.atomic_append_with_rename('test_atomic.txt', '\n这是追加的第二行内容\n')
print(f"结果: {success}, {message}")
# 验证追加结果
with open('test_atomic.txt', 'r') as f:
content = f.read()
print(f"追加后内容: {repr(content)}")
demo_basic_atomic_rename()
---
02.并发写入策略
a.并发问题分析
a.竞态条件
多个进程同时创建临时文件可能导致命名冲突。
b.目录锁
使用目录级别的锁来协调并发写入。
c.进程标识
利用进程ID和时间戳生成唯一临时文件名。
b.并发控制实现
a.文件锁机制
使用操作系统文件锁保证原子性。
b.时间戳排序
基于时间戳的写入顺序控制。
c.版本管理
维护文件版本号处理并发修改。
c.并发写入示例
---
# 并发安全的临时文件写入示例
import threading
import time
import uuid
from concurrent.futures import ThreadPoolExecutor
import fcntl # Unix文件锁
class ConcurrentAtomicWriter:
def __init__(self, base_dir=None):
self.base_dir = base_dir or os.getcwd()
self.lock_files = {}
self.writing_processes = set()
def _get_process_lock(self, target_path):
"""获取进程级文件锁"""
lock_path = target_path + '.process.lock'
# 使用UUID+进程ID+时间戳确保唯一性
lock_id = f"{os.getpid()}_{uuid.uuid4().hex}_{int(time.time())}"
try:
# 创建锁文件（原子操作）
with open(lock_path, 'w') as f:
f.write(lock_id)
return lock_id, lock_path
except FileExistsError:
return None, None
def _release_process_lock(self, lock_path, lock_id):
"""释放进程级文件锁"""
try:
if os.path.exists(lock_path):
with open(lock_path, 'r') as f:
stored_id = f.read().strip()
# 只有锁的所有者才能释放
if stored_id == lock_id:
os.remove(lock_path)
return True
return False
except:
return False
def _wait_for_lock_release(self, target_path, max_wait=5.0):
"""等待其他进程释放锁"""
lock_path = target_path + '.process.lock'
start_time = time.time()
while os.path.exists(lock_path) and (time.time() - start_time) < max_wait:
time.sleep(0.01)
return not os.path.exists(lock_path)
def concurrent_atomic_write(self, target_path, data, writer_id=None):
"""并发安全的原子写入"""
if writer_id is None:
writer_id = f"{threading.current_thread().name}_{os.getpid()}"
# 等待其他进程完成
if not self._wait_for_lock_release(target_path):
return False, f"等待锁超时: {target_path}"
# 获取进程锁
lock_id, lock_path = self._get_process_lock(target_path)
if not lock_id:
return False, f"无法获取锁: {target_path}"
try:
# 创建唯一临时文件
temp_dir = os.path.dirname(target_path) or self.base_dir
temp_prefix = f".conc_{writer_id}_{int(time.time())}_"
fd, temp_path = tempfile.mkstemp(
dir=temp_dir,
prefix=temp_prefix,
suffix='.tmp'
)
os.close(fd)
# 写入数据
with open(temp_path, 'w', encoding='utf-8') as f:
f.write(f"# Writer: {writer_id}\n")
f.write(f"# Timestamp: {time.time()}\n")
f.write(str(data))
# 原子重命名
backup_path = target_path + f".{writer_id}.bak"
if os.path.exists(target_path):
shutil.move(target_path, backup_path)
shutil.move(temp_path, target_path)
return True, f"并发写入成功: {writer_id}"
except Exception as e:
# 清理临时文件
if 'temp_path' in locals() and os.path.exists(temp_path):
try:
os.remove(temp_path)
except:
pass
return False, f"并发写入失败: {e}"
finally:
# 释放锁
self._release_process_lock(lock_path, lock_id)
def batch_concurrent_writes(self, target_path, data_list, num_workers=4):
"""批量并发写入测试"""
def writer_worker(worker_id, data):
"""工作线程函数"""
return self.concurrent_atomic_write(
target_path, data, f"worker_{worker_id}"
)
# 使用线程池执行并发写入
with ThreadPoolExecutor(max_workers=num_workers) as executor:
futures = []
for i, data in enumerate(data_list):
future = executor.submit(writer_worker, i, data)
futures.append(future)
results = []
for i, future in enumerate(futures):
try:
success, message = future.result(timeout=10)
results.append((i, success, message))
except Exception as e:
results.append((i, False, f"执行异常: {e}"))
return results
def test_concurrent_atomic_writes():
"""测试并发原子写入"""
print("=== 并发原子写入测试 ===")
writer = ConcurrentAtomicWriter()
# 测试数据
test_messages = [
"第一条消息来自Worker 0",
"第二条消息来自Worker 1",
"第三条消息来自Worker 2",
"第三条消息来自Worker 3",
"第五条消息来自Worker 4"
]
# 执行并发写入
print("执行并发写入...")
results = writer.batch_concurrent_writes(
'concurrent_output.txt', test_messages, num_workers=5
)
# 显示结果
print("\n并发写入结果:")
for worker_id, success, message in results:
status = "成功" if success else "失败"
print(f" Worker {worker_id}: {status} - {message}")
# 检查最终文件
if os.path.exists('concurrent_output.txt'):
print(f"\n最终文件内容:")
with open('concurrent_output.txt', 'r', encoding='utf-8') as f:
content = f.read()
print(content)
else:
print("\n目标文件不存在")
test_concurrent_atomic_writes()
---
03.备份与恢复机制
a.备份策略
a.写入前备份
在修改前创建原文件的备份。
b.版本控制
维护多个历史版本用于回滚。
c.增量备份
只备份变更的部分以节省空间。
b.恢复机制
a.自动恢复
检测到错误时自动从备份恢复。
b.手动恢复
提供工具进行手动版本回滚。
c.完整性验证
验证恢复后的文件完整性。
c.备份恢复示例
---
# 备份与恢复机制的完整实现
import json
import shutil
import glob
from datetime import datetime, timedelta
class BackupRestoreManager:
def __init__(self, backup_dir='backups', max_versions=10):
self.backup_dir = backup_dir
self.max_versions = max_versions
os.makedirs(backup_dir, exist_ok=True)
def _get_backup_path(self, target_path, version=None):
"""生成备份文件路径"""
target_name = os.path.basename(target_path)
timestamp = datetime.now().strftime('%Y%m%d_%H%M%S_%f')[:-3]
if version is not None:
backup_name = f"{target_name}.v{version}.{timestamp}.bak"
else:
backup_name = f"{target_name}.{timestamp}.bak"
return os.path.join(self.backup_dir, backup_name)
def _get_version_number(self, target_path):
"""获取下一个版本号"""
pattern = os.path.join(self.backup_dir, f"{os.path.basename(target_path)}.*.bak")
existing_backups = glob.glob(pattern)
version_numbers = []
for backup in existing_backups:
try:
parts = os.path.basename(backup).split('.')
if 'v' in parts[1]:
version = int(parts[1][1:])
version_numbers.append(version)
except:
continue
return max(version_numbers, default=0) + 1
def create_backup(self, target_path, versioned=True):
"""创建文件备份"""
if not os.path.exists(target_path):
return False, f"源文件不存在: {target_path}"
try:
if versioned:
version = self._get_version_number(target_path)
backup_path = self._get_backup_path(target_path, version)
else:
backup_path = self._get_backup_path(target_path)
# 复制文件
shutil.copy2(target_path, backup_path)
# 创建备份元数据
metadata = {
'original_path': target_path,
'backup_path': backup_path,
'timestamp': datetime.now().isoformat(),
'size': os.path.getsize(target_path),
'version': version if versioned else None
}
metadata_path = backup_path + '.meta'
with open(metadata_path, 'w', encoding='utf-8') as f:
json.dump(metadata, f, indent=2, ensure_ascii=False)
# 清理旧版本
self._cleanup_old_versions(target_path)
return True, backup_path
except Exception as e:
return False, f"备份失败: {e}"
def _cleanup_old_versions(self, target_path):
"""清理旧版本备份"""
target_name = os.path.basename(target_path)
pattern = os.path.join(self.backup_dir, f"{target_name}.*.bak")
existing_backups = glob.glob(pattern)
if len(existing_backups) <= self.max_versions:
return
# 按时间排序，删除最旧的
existing_backups.sort(key=os.path.getctime)
to_delete = existing_backups[:-self.max_versions]
for backup in to_delete:
try:
os.remove(backup)
# 删除元数据文件
meta_path = backup + '.meta'
if os.path.exists(meta_path):
os.remove(meta_path)
except Exception as e:
print(f"删除旧备份失败 {backup}: {e}")
def atomic_write_with_backup(self, target_path, data):
"""带备份的原子写入"""
# 1. 创建备份
if os.path.exists(target_path):
backup_success, backup_path = self.create_backup(target_path)
if not backup_success:
return False, f"创建备份失败: {backup_path}"
# 2. 原子写入新数据
try:
temp_fd, temp_path = tempfile.mkstemp(
dir=os.path.dirname(target_path) or '.',
suffix='.tmp'
)
os.close(temp_fd)
with open(temp_path, 'w', encoding='utf-8') as f:
if isinstance(data, str):
f.write(data)
else:
f.write(str(data))
# 确保数据写入磁盘
f.flush()
os.fsync(f.fileno())
# 原子重命名
if os.path.exists(target_path):
os.remove(target_path)
shutil.move(temp_path, target_path)
return True, f"写入成功，已备份"
except Exception as e:
# 清理临时文件
if 'temp_path' in locals() and os.path.exists(temp_path):
try:
os.remove(temp_path)
except:
pass
return False, f"写入失败: {e}"
def list_backups(self, target_path):
"""列出所有备份"""
target_name = os.path.basename(target_path)
pattern = os.path.join(self.backup_dir, f"{target_name}.*.bak")
backup_files = glob.glob(pattern)
backups = []
for backup in backup_files:
try:
meta_path = backup + '.meta'
if os.path.exists(meta_path):
with open(meta_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
backups.append(metadata)
else:
# 没有元数据文件的基本信息
backups.append({
'original_path': target_path,
'backup_path': backup,
'timestamp': datetime.fromtimestamp(os.path.getctime(backup)).isoformat(),
'size': os.path.getsize(backup),
'version': None
})
except:
continue
return sorted(backups, key=lambda x: x['timestamp'], reverse=True)
def restore_from_backup(self, target_path, backup_path=None, version=None):
"""从备份恢复文件"""
try:
if backup_path is None:
# 查找指定版本的备份
backups = self.list_backups(target_path)
if version is not None:
matching = [b for b in backups if b.get('version') == version]
if not matching:
return False, f"版本 {version} 的备份不存在"
backup_path = matching[0]['backup_path']
elif backups:
backup_path = backups[0]['backup_path'] # 最新备份
else:
return False, "没有可用的备份"
if not os.path.exists(backup_path):
return False, f"备份文件不存在: {backup_path}"
# 创建当前文件的备份
if os.path.exists(target_path):
self.create_backup(target_path, versioned=False)
# 恢复文件
shutil.copy2(backup_path, target_path)
return True, f"已从备份恢复: {os.path.basename(backup_path)}"
except Exception as e:
return False, f"恢复失败: {e}"
def verify_backup_integrity(self, backup_path):
"""验证备份文件完整性"""
try:
if not os.path.exists(backup_path):
return False, "备份文件不存在"
# 检查文件大小
if os.path.getsize(backup_path) == 0:
return False, "备份文件为空"
# 计算校验和
with open(backup_path, 'rb') as f:
content = f.read()
if not content:
return False, "备份文件内容为空"
# 检查元数据
meta_path = backup_path + '.meta'
if os.path.exists(meta_path):
with open(meta_path, 'r', encoding='utf-8') as f:
metadata = json.load(f)
# 验证文件大小匹配
if metadata.get('size') != len(content):
return False, "备份文件大小与元数据不匹配"
return True, "备份文件完整性验证通过"
except Exception as e:
return False, f"验证失败: {e}"
def test_backup_restore_mechanism():
"""测试备份恢复机制"""
print("=== 备份恢复机制测试 ===")
manager = BackupRestoreManager('test_backups', max_versions=3)
# 1. 创建初始文件
print("\n1. 创建初始文件:")
initial_data = "初始版本的数据内容\n"
success, message = manager.atomic_write_with_backup('test_file.txt', initial_data)
print(f"结果: {success}, {message}")
# 2. 第一次更新
print("\n2. 第一次更新:")
update1_data = "第一次更新的数据内容\n" + initial_data
success, message = manager.atomic_write_with_backup('test_file.txt', update1_data)
print(f"结果: {success}, {message}")
# 3. 第二次更新
print("\n3. 第二次更新:")
update2_data = "第二次更新的数据内容\n" + update1_data
success, message = manager.atomic_write_with_backup('test_file.txt', update2_data)
print(f"结果: {success}, {message}")
# 4. 列出所有备份
print("\n4. 备份列表:")
backups = manager.list_backups('test_file.txt')
for i, backup in enumerate(backups, 1):
print(f" 备份 {i}: 版本={backup.get('version')}, 时间={backup.get('timestamp')[:19]}, 大小={backup.get('size')}字节")
# 5. 验证备份完整性
print("\n5. 验证备份完整性:")
for backup in backups:
backup_path = backup['backup_path']
is_valid, message = manager.verify_backup_integrity(backup_path)
print(f" {os.path.basename(backup_path)}: {is_valid}, {message}")
# 6. 恢复到第一个版本
print("\n6. 恢复到第一个版本:")
success, message = manager.restore_from_backup('test_file.txt', version=1)
print(f"结果: {success}, {message}")
# 验证恢复结果
with open('test_file.txt', 'r', encoding='utf-8') as f:
restored_content = f.read()
print(f"恢复后内容: {repr(restored_content[:50])}")
test_backup_restore_mechanism()
---
8.3 写前日志WAL
01.WAL机制原理
a.写前日志概念
在修改数据之前，先将操作记录到日志文件中，确保操作的持久性和可恢复性。
b.核心机制
a.先写日志
任何数据修改操作都必须先写入日志。
b.顺序写入
日志按时间顺序追加写入，保证操作的时序性。
c.检查点
定期创建检查点，标记已经安全持久化的状态。
c.优势分析
a.故障恢复
系统崩溃后可以通过重放日志恢复数据一致性。
b.数据持久性
确保已提交的操作不会丢失。
c.性能优化
顺序写入日志比随机修改数据文件更高效。
d.WAL基础实现
---
# 写前日志(WAL)基础实现
import os
import time
import json
import struct
from datetime import datetime
from typing import Dict, List, Any, Optional, Tuple
class WALManager:
def __init__(self, wal_dir: str = 'wal_logs', max_log_size: int = 10 * 1024 * 1024):
self.wal_dir = wal_dir
self.max_log_size = max_log_size
self.current_log_file = None
self.log_sequence = 0
self.last_checkpoint = 0
# 确保WAL目录存在
os.makedirs(wal_dir, exist_ok=True)
# 初始化或恢复WAL状态
self._initialize_wal()
def _initialize_wal(self):
"""初始化WAL状态"""
# 查找最新的日志文件
log_files = self._get_log_files()
if log_files:
self.current_log_file = log_files[-1] # 最新的日志文件
self.log_sequence = self._extract_sequence_from_filename(self.current_log_file)
else:
# 创建新的日志文件
self._create_new_log_file()
# 读取最后检查点信息
self._load_checkpoint()
def _get_log_files(self) -> List[str]:
"""获取所有WAL日志文件，按序列号排序"""
log_files = []
for filename in os.listdir(self.wal_dir):
if filename.startswith('wal_') and filename.endswith('.log'):
log_files.append(os.path.join(self.wal_dir, filename))
log_files.sort(key=lambda x: self._extract_sequence_from_filename(x))
return log_files
def _extract_sequence_from_filename(self, filepath: str) -> int:
"""从文件名中提取序列号"""
filename = os.path.basename(filepath)
try:
# wal_000001.log -> 1
sequence_part = filename[4:-4] # 去掉 'wal_' 和 '.log'
return int(sequence_part)
except:
return 0
def _create_new_log_file(self):
"""创建新的WAL日志文件"""
self.log_sequence += 1
filename = f"wal_{self.log_sequence:06d}.log"
self.current_log_file = os.path.join(self.wal_dir, filename)
# 创建空的日志文件，写入文件头
with open(self.current_log_file, 'wb') as f:
header = struct.pack('!I', 0x57414C00) # 'WAL\0' magic number
f.write(header)
f.write(struct.pack('!Q', int(time.time()))) # 创建时间戳
f.write(struct.pack('!Q', 0)) # 初始记录数
def _load_checkpoint(self):
"""加载检查点信息"""
checkpoint_file = os.path.join(self.wal_dir, 'checkpoint.json')
if os.path.exists(checkpoint_file):
try:
with open(checkpoint_file, 'r', encoding='utf-8') as f:
checkpoint_data = json.load(f)
self.last_checkpoint = checkpoint_data.get('last_log_sequence', 0)
except:
self.last_checkpoint = 0
def _write_log_entry(self, operation: str, key: str, value: Any, old_value: Any = None) -> int:
"""写入日志条目"""
if not self.current_log_file:
self._create_new_log_file()
# 检查日志文件大小，如果超过限制则创建新文件
if os.path.getsize(self.current_log_file) > self.max_log_size:
self._create_new_log_file()
# 构造日志条目
timestamp = int(time.time() * 1000000) # 微秒级时间戳
entry_data = {
'timestamp': timestamp,
'operation': operation,
'key': key,
'value': value,
'old_value': old_value
}
# 序列化日志条目
entry_json = json.dumps(entry_data, ensure_ascii=False)
entry_bytes = entry_json.encode('utf-8')
# 写入日志文件 (操作类型, 数据长度, 数据内容)
operation_code = {'SET': 1, 'DELETE': 2, 'UPDATE': 3}.get(operation, 0)
entry_header = struct.pack('!BHQ', operation_code, len(key), len(entry_bytes))
with open(self.current_log_file, 'ab') as f:
f.write(entry_header)
f.write(key.encode('utf-8'))
f.write(entry_bytes)
f.flush()
os.fsync(f.fileno()) # 强制写入磁盘
return self.log_sequence
def log_operation(self, operation: str, key: str, value: Any = None, old_value: Any = None) -> bool:
"""记录操作到WAL"""
try:
self._write_log_entry(operation, key, value, old_value)
return True
except Exception as e:
print(f"WAL写入失败: {e}")
return False
def create_checkpoint(self, checkpoint_data: Dict[str, Any]) -> bool:
"""创建检查点"""
try:
checkpoint_file = os.path.join(self.wal_dir, 'checkpoint.json')
checkpoint_info = {
'timestamp': datetime.now().isoformat(),
'last_log_sequence': self.log_sequence,
'checkpoint_data': checkpoint_data
}
# 原子写入检查点文件
temp_file = checkpoint_file + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
json.dump(checkpoint_info, f, indent=2, ensure_ascii=False)
if os.path.exists(checkpoint_file):
os.remove(checkpoint_file)
os.rename(temp_file, checkpoint_file)
self.last_checkpoint = self.log_sequence
return True
except Exception as e:
print(f"创建检查点失败: {e}")
return False
def recover_from_wal(self) -> List[Dict[str, Any]]:
"""从WAL恢复操作"""
recovered_operations = []
try:
# 加载最新检查点
checkpoint_file = os.path.join(self.wal_dir, 'checkpoint.json')
checkpoint_data = {}
if os.path.exists(checkpoint_file):
with open(checkpoint_file, 'r', encoding='utf-8') as f:
checkpoint_info = json.load(f)
checkpoint_data = checkpoint_info.get('checkpoint_data', {})
self.last_checkpoint = checkpoint_info.get('last_log_sequence', 0)
# 重放检查点之后的日志
log_files = self._get_log_files()
for log_file in log_files:
file_sequence = self._extract_sequence_from_filename(log_file)
if file_sequence <= self.last_checkpoint:
continue
operations = self._parse_log_file(log_file)
recovered_operations.extend(operations)
except Exception as e:
print(f"WAL恢复失败: {e}")
return recovered_operations
def _parse_log_file(self, log_file: str) -> List[Dict[str, Any]]:
"""解析单个日志文件"""
operations = []
try:
with open(log_file, 'rb') as f:
# 跳过文件头
f.seek(16) # magic(4) + timestamp(8) + record_count(8)
while True:
# 读取条目头
header = f.read(9) # operation(1) + key_len(4) + data_len(8)
if len(header) < 9:
break
operation_code, key_len, data_len = struct.unpack('!BHQ', header)
# 读取键名
key_bytes = f.read(key_len)
if len(key_bytes) < key_len:
break
key = key_bytes.decode('utf-8')
# 读取数据
data_bytes = f.read(data_len)
if len(data_bytes) < data_len:
break
entry_data = json.loads(data_bytes.decode('utf-8'))
operations.append(entry_data)
except Exception as e:
print(f"解析日志文件失败 {log_file}: {e}")
return operations
def cleanup_old_logs(self, keep_last_n: int = 3) -> int:
"""清理旧的日志文件"""
log_files = self._get_log_files()
if len(log_files) <= keep_last_n:
return 0
# 保留最后N个文件，删除其余的
to_delete = log_files[:-keep_last_n]
deleted_count = 0
for log_file in to_delete:
try:
os.remove(log_file)
deleted_count += 1
except Exception as e:
print(f"删除旧日志文件失败 {log_file}: {e}")
return deleted_count
def demo_wal_basic_operations():
"""演示WAL基础操作"""
print("=== WAL基础操作演示 ===")
wal = WALManager('demo_wal')
# 1. 记录一些操作
print("\n1. 记录操作:")
operations = [
('SET', 'user:1', {'name': 'Alice', 'age': 25}),
('SET', 'user:2', {'name': 'Bob', 'age': 30}),
('UPDATE', 'user:1', {'age': 26}, {'age': 25}),
('DELETE', 'user:2', None, {'name': 'Bob', 'age': 30})
]
for op in operations:
success = wal.log_operation(*op)
print(f" {op[0]} {op[1]}: {'成功' if success else '失败'}")
# 2. 创建检查点
print("\n2. 创建检查点:")
checkpoint_data = {
'user_count': 1,
'last_modified': time.time(),
'users': {'user:1': {'name': 'Alice', 'age': 26}}
}
success = wal.create_checkpoint(checkpoint_data)
print(f" 检查点创建: {'成功' if success else '失败'}")
# 3. 模拟恢复
print("\n3. 模拟WAL恢复:")
recovered_ops = wal.recover_from_wal()
print(f" 恢复的操作数量: {len(recovered_ops)}")
for i, op in enumerate(recovered_ops):
print(f" {i+1}. {op.get('operation')} {op.get('key')} at {op.get('timestamp')}")
# 4. 清理旧日志
print("\n4. 清理旧日志:")
deleted = wal.cleanup_old_logs(keep_last_n=1)
print(f" 删除的旧日志文件数量: {deleted}")
demo_wal_basic_operations()
---
02.高性能WAL实现
a.性能优化策略
a.批量写入
将多个日志条目批量写入，减少I/O操作次数。
b.缓冲区管理
使用内存缓冲区缓存日志条目，定期刷盘。
c.异步写入
使用独立线程执行日志写入操作。
b.高级特性
a.压缩存储
对日志条目进行压缩以节省存储空间。
b.索引结构
建立日志索引加速恢复过程。
c.并行恢复
使用多线程并行处理日志恢复。
c.高性能实现示例
---
# 高性能WAL实现
import threading
import queue
import zlib
import heapq
from concurrent.futures import ThreadPoolExecutor
from collections import defaultdict
class HighPerformanceWAL:
def __init__(self, wal_dir: str = 'hp_wal',
buffer_size: int = 64 * 1024,
flush_interval: float = 0.1,
compression_level: int = 6):
self.wal_dir = wal_dir
self.buffer_size = buffer_size
self.flush_interval = flush_interval
self.compression_level = compression_level
# 日志缓冲区
self.write_buffer = []
self.buffer_lock = threading.Lock()
self.buffer_event = threading.Event()
# 异步写入线程
self.writer_thread = None
self.stop_writer = False
# 索引结构
self.log_index = {} # key -> [(log_file, offset, timestamp), ...]
self.index_lock = threading.Lock()
# 性能统计
self.stats = {
'writes_queued': 0,
'writes_completed': 0,
'bytes_written': 0,
'compression_ratio': 0.0
}
# 初始化
os.makedirs(wal_dir, exist_ok=True)
self._start_writer_thread()
def _start_writer_thread(self):
"""启动异步写入线程"""
self.writer_thread = threading.Thread(target=self._writer_worker, daemon=True)
self.writer_thread.start()
def _writer_worker(self):
"""异步写入工作线程"""
while not self.stop_writer:
try:
# 等待缓冲区有数据或停止信号
self.buffer_event.wait(timeout=self.flush_interval)
if self.stop_writer:
break
# 获取缓冲区数据
with self.buffer_lock:
if not self.write_buffer:
self.buffer_event.clear()
continue
entries_to_write = self.write_buffer[:]
self.write_buffer.clear()
self.buffer_event.clear()
# 批量写入
if entries_to_write:
self._batch_write_entries(entries_to_write)
except Exception as e:
print(f"写入线程异常: {e}")
def _batch_write_entries(self, entries):
"""批量写入日志条目"""
if not entries:
return
try:
# 按时间戳排序
entries.sort(key=lambda x: x['timestamp'])
# 压缩数据
uncompressed_data = json.dumps(entries, ensure_ascii=False).encode('utf-8')
compressed_data = zlib.compress(uncompressed_data, self.compression_level)
# 写入文件
log_file = self._get_current_log_file()
with open(log_file, 'ab') as f:
# 写入压缩块头
block_header = struct.pack('!HQ', len(compressed_data), len(uncompressed_data))
f.write(block_header)
f.write(compressed_data)
f.flush()
os.fsync(f.fileno())
# 更新索引
self._update_index(entries, log_file)
# 更新统计
with self.buffer_lock:
self.stats['writes_completed'] += len(entries)
self.stats['bytes_written'] += len(compressed_data)
if uncompressed_data:
ratio = len(compressed_data) / len(uncompressed_data)
self.stats['compression_ratio'] = ratio
except Exception as e:
print(f"批量写入失败: {e}")
def _get_current_log_file(self):
"""获取当前日志文件路径"""
# 简化实现，每天创建一个新文件
today = datetime.now().strftime('%Y%m%d')
return os.path.join(self.wal_dir, f'wal_{today}.log')
def _update_index(self, entries, log_file):
"""更新日志索引"""
with self.index_lock:
for entry in entries:
key = entry.get('key')
if key:
if key not in self.log_index:
self.log_index[key] = []
self.log_index[key].append((log_file, entry['timestamp']))
def log_operation_async(self, operation: str, key: str, value: Any = None, old_value: Any = None) -> bool:
"""异步记录操作到WAL"""
entry = {
'timestamp': int(time.time() * 1000000),
'operation': operation,
'key': key,
'value': value,
'old_value': old_value
}
try:
with self.buffer_lock:
self.write_buffer.append(entry)
self.stats['writes_queued'] += 1
self.buffer_event.set()
return True
except Exception as e:
print(f"异步日志记录失败: {e}")
return False
def flush_buffer(self, timeout: float = 5.0) -> bool:
"""强制刷新缓冲区"""
start_time = time.time()
while True:
with self.buffer_lock:
if not self.write_buffer:
return True
if time.time() - start_time > timeout:
return False
time.sleep(0.01)
def parallel_recover(self, num_workers: int = 4) -> Tuple[List[Dict], Dict[str, Any]]:
"""并行恢复WAL日志"""
log_files = []
for filename in os.listdir(self.wal_dir):
if filename.startswith('wal_') and filename.endswith('.log'):
log_files.append(os.path.join(self.wal_dir, filename))
if not log_files:
return [], {}
# 并行解析日志文件
with ThreadPoolExecutor(max_workers=num_workers) as executor:
futures = [executor.submit(self._parse_compressed_log, log_file) for log_file in log_files]
all_operations = []
for future in futures:
try:
operations = future.result(timeout=30)
all_operations.extend(operations)
except Exception as e:
print(f"并行解析日志失败: {e}")
# 按时间戳排序
all_operations.sort(key=lambda x: x['timestamp'])
# 构建最新状态
latest_state = {}
for op in all_operations:
key = op['key']
operation = op['operation']
if operation == 'SET' or operation == 'UPDATE':
latest_state[key] = op['value']
elif operation == 'DELETE':
latest_state.pop(key, None)
return all_operations, latest_state
def _parse_compressed_log(self, log_file):
"""解析压缩的日志文件"""
operations = []
try:
with open(log_file, 'rb') as f:
while True:
# 读取块头
header = f.read(10) # compressed_len(4) + uncompressed_len(8)
if len(header) < 10:
break
compressed_len, uncompressed_len = struct.unpack('!HQ', header)
# 读取压缩数据
compressed_data = f.read(compressed_len)
if len(compressed_data) < compressed_len:
break
# 解压缩
uncompressed_data = zlib.decompress(compressed_data)
entries = json.loads(uncompressed_data.decode('utf-8'))
operations.extend(entries)
except Exception as e:
print(f"解析压缩日志失败 {log_file}: {e}")
return operations
def get_performance_stats(self) -> Dict[str, Any]:
"""获取性能统计信息"""
with self.buffer_lock:
stats = self.stats.copy()
stats['buffer_size'] = len(self.write_buffer)
return stats
def shutdown(self):
"""关闭WAL，确保所有数据写入磁盘"""
print("正在关闭WAL...")
self.stop_writer = True
self.buffer_event.set()
if self.writer_thread and self.writer_thread.is_alive():
self.writer_thread.join(timeout=10)
# 最后一次刷新缓冲区
self.flush_buffer()
print("WAL已关闭")
def demo_high_performance_wal():
"""演示高性能WAL"""
print("=== 高性能WAL演示 ===")
wal = HighPerformanceWAL('hp_demo_wal', buffer_size=32*1024)
# 1. 异步写入大量操作
print("\n1. 异步写入大量操作:")
start_time = time.time()
for i in range(1000):
operation = 'SET' if i % 3 != 0 else 'UPDATE'
key = f'item_{i}'
value = {'id': i, 'data': f'data_{i}' * 10} # 较大的数据
wal.log_operation_async(operation, key, value)
write_time = time.time() - start_time
print(f" 异步写入1000个操作耗时: {write_time:.4f}秒")
# 2. 等待写入完成
print("\n2. 等待写入完成:")
flushed = wal.flush_buffer(timeout=10)
print(f" 缓冲区刷新: {'成功' if flushed else '超时'}")
# 3. 并行恢复
print("\n3. 并行恢复:")
start_time = time.time()
recovered_ops, final_state = wal.parallel_recover(num_workers=4)
recovery_time = time.time() - start_time
print(f" 恢复的操作数量: {len(recovered_ops)}")
print(f" 最终状态键数量: {len(final_state)}")
print(f" 并行恢复耗时: {recovery_time:.4f}秒")
# 4. 性能统计
print("\n4. 性能统计:")
stats = wal.get_performance_stats()
for key, value in stats.items():
print(f" {key}: {value}")
# 关闭WAL
wal.shutdown()
demo_high_performance_wal()
---
03.WAL应用场景
a.数据库系统
a.事务日志
记录数据库事务的完整操作序列。
b.崩溃恢复
系统崩溃后通过重放日志恢复数据一致性。
c.复制同步
使用WAL实现主从数据库的数据同步。
b.消息队列
a.持久化保证
确保消息的持久化存储和可靠传递。
b.顺序保证
维护消息的严格顺序性。
c.重放机制
支持消息的重新处理和故障恢复。
c.配置管理
a.配置变更日志
记录系统配置的变更历史。
b.回滚支持
支持配置的版本管理和回滚。
c.审计追踪
提供完整的配置变更审计记录。
d.WAL应用示例
---
# WAL应用场景示例 - 简单的键值存储
class KeyValueStoreWithWAL:
def __init__(self, data_dir: str = 'kv_store'):
self.data_dir = data_dir
self.wal_dir = os.path.join(data_dir, 'wal')
self.data_file = os.path.join(data_dir, 'data.json')
# 内存数据存储
self.data = {}
self.data_lock = threading.RLock()
# WAL管理器
self.wal = WALManager(self.wal_dir)
# 启动时恢复数据
self._recover_data()
def _recover_data(self):
"""从WAL和快照恢复数据"""
print("正在恢复数据...")
# 1. 加载最新快照
if os.path.exists(self.data_file):
try:
with open(self.data_file, 'r', encoding='utf-8') as f:
self.data = json.load(f)
print(f"从快照加载了 {len(self.data)} 个键")
except Exception as e:
print(f"加载快照失败: {e}")
self.data = {}
# 2. 重放WAL日志
recovered_ops = self.wal.recover_from_wal()
applied_ops = 0
with self.data_lock:
for op in recovered_ops:
try:
self._apply_operation(op, skip_wal=True)
applied_ops += 1
except Exception as e:
print(f"应用操作失败: {e}")
print(f"重放了 {applied_ops} 个WAL操作")
def _apply_operation(self, operation, skip_wal=False):
"""应用单个操作到数据存储"""
with self.data_lock:
op_type = operation['operation']
key = operation['key']
value = operation.get('value')
old_value = operation.get('old_value')
# 验证操作一致性
current_value = self.data.get(key)
if op_type in ['UPDATE', 'DELETE'] and current_value != old_value:
raise ValueError(f"操作一致性验证失败: key={key}")
if op_type == 'SET' or op_type == 'UPDATE':
self.data[key] = value
elif op_type == 'DELETE':
self.data.pop(key, None)
def set(self, key: str, value: Any) -> bool:
"""设置键值"""
try:
with self.data_lock:
old_value = self.data.get(key)
operation = 'UPDATE' if old_value is not None else 'SET'
# 记录WAL
if not self.wal.log_operation(operation, key, value, old_value):
return False
# 应用操作
self.data[key] = value
return True
except Exception as e:
print(f"SET操作失败: {e}")
return False
def get(self, key: str, default: Any = None) -> Any:
"""获取键值"""
with self.data_lock:
return self.data.get(key, default)
def delete(self, key: str) -> bool:
"""删除键"""
try:
with self.data_lock:
if key not in self.data:
return True # 键不存在，认为删除成功
old_value = self.data[key]
# 记录WAL
if not self.wal.log_operation('DELETE', key, None, old_value):
return False
# 应用操作
del self.data[key]
return True
except Exception as e:
print(f"DELETE操作失败: {e}")
return False
def create_snapshot(self) -> bool:
"""创建数据快照"""
try:
with self.data_lock:
snapshot_data = {
'timestamp': datetime.now().isoformat(),
'data': dict(self.data),
'key_count': len(self.data)
}
# 原子写入快照文件
temp_file = self.data_file + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
json.dump(snapshot_data['data'], f, indent=2, ensure_ascii=False)
if os.path.exists(self.data_file):
os.remove(self.data_file)
os.rename(temp_file, self.data_file)
# 创建WAL检查点
checkpoint_data = {
'snapshot_timestamp': snapshot_data['timestamp'],
'key_count': snapshot_data['key_count']
}
self.wal.create_checkpoint(checkpoint_data)
return True
except Exception as e:
print(f"创建快照失败: {e}")
return False
def compact_wal(self) -> bool:
"""压缩WAL日志"""
try:
# 1. 创建快照
if not self.create_snapshot():
return False
# 2. 清理旧日志
deleted = self.wal.cleanup_old_logs(keep_last_n=1)
print(f"清理了 {deleted} 个旧WAL文件")
return True
except Exception as e:
print(f"WAL压缩失败: {e}")
return False
def get_stats(self) -> Dict[str, Any]:
"""获取存储统计信息"""
with self.data_lock:
stats = {
'key_count': len(self.data),
'data_size': len(json.dumps(self.data, ensure_ascii=False)),
'wal_files': len(self.wal._get_log_files())
}
return stats
def demo_kv_store_with_wal():
"""演示带WAL的键值存储"""
print("=== 键值存储WAL应用演示 ===")
# 1. 创建存储实例
store = KeyValueStoreWithWAL('demo_kv_store')
# 2. 添加数据
print("\n2. 添加数据:")
test_data = {
'user:1': {'name': 'Alice', 'age': 25, 'email': '[email protected]'},
'user:2': {'name': 'Bob', 'age': 30, 'email': '[email protected]'},
'config:app': {'version': '1.0.0', 'debug': False},
'session:abc123': {'user_id': 1, 'created': time.time()}
}
for key, value in test_data.items():
success = store.set(key, value)
print(f" SET {key}: {'成功' if success else '失败'}")
# 3. 读取数据
print("\n3. 读取数据:")
for key in test_data.keys():
value = store.get(key)
print(f" GET {key}: {value}")
# 4. 更新数据
print("\n4. 更新数据:")
updates = {
'user:1': {'name': 'Alice', 'age': 26, 'email': '[email protected]'},
'config:app': {'version': '1.1.0', 'debug': True}
}
for key, value in updates.items():
success = store.set(key, value)
print(f" UPDATE {key}: {'成功' if success else '失败'}")
# 5. 删除数据
print("\n5. 删除数据:")
success = store.delete('session:abc123')
print(f" DELETE session:abc123: {'成功' if success else '失败'}")
# 6. 显示统计信息
print("\n6. 存储统计:")
stats = store.get_stats()
for key, value in stats.items():
print(f" {key}: {value}")
# 7. 创建快照
print("\n7. 创建快照:")
success = store.create_snapshot()
print(f" 快照创建: {'成功' if success else '失败'}")
# 8. 模拟崩溃重启
print("\n8. 模拟崩溃重启:")
del store # 销毁当前实例
new_store = KeyValueStoreWithWAL('demo_kv_store') # 重新创建实例
print("重启后的存储统计:")
new_stats = new_store.get_stats()
for key, value in new_stats.items():
print(f" {key}: {value}")
# 验证数据完整性
print("\n验证数据完整性:")
for key in ['user:1', 'user:2', 'config:app']:
old_value = test_data.get(key)
if key in updates:
old_value = updates[key]
if key == 'session:abc123':
continue # 已删除
new_value = new_store.get(key)
integrity_check = new_value == old_value
print(f" {key}: {'✓' if integrity_check else '✗'}")
demo_kv_store_with_wal()
---
8.4 文件锁配合
01.文件锁机制原理
a.锁类型
a.共享锁(读锁)
多个进程可以同时持有，用于读操作。
b.排他锁(写锁)
只能被一个进程持有，用于写操作。
c.建议性锁
依赖进程自觉遵守，不强制执行。
b.锁实现方式
a.fcntl系统调用
Unix/Linux系统的标准文件锁实现。
b.msflock函数
跨平台的文件锁函数。
c.锁文件机制
通过创建/删除锁文件实现简单锁机制。
c.锁的特点
a.原子性
获取和释放锁的操作是原子的。
b.阻塞与非阻塞
支持阻塞等待和非阻塞获取锁。
c.死锁预防
需要设计合理的锁获取顺序避免死锁。
d.基础文件锁实现
---
# 基础文件锁实现
import os
import fcntl
import time
import threading
from contextlib import contextmanager
from typing import Optional, Any
class FileLockManager:
def __init__(self, timeout: float = 10.0):
self.timeout = timeout
self.locked_files = {} # 文件路径 -> (文件对象, 锁类型)
self.lock_stack = {} # 线程ID -> 锁栈，用于重入
@contextmanager
def acquire_lock(self, filepath: str, lock_type: str = 'exclusive', blocking: bool = True):
"""
获取文件锁
Args:
filepath: 文件路径
lock_type: 'shared' 或 'exclusive'
blocking: 是否阻塞等待
"""
thread_id = threading.current_thread().ident
# 检查是否已持有该锁（重入锁）
if thread_id in self.lock_stack and filepath in self.lock_stack[thread_id]:
# 增加重入计数
self.lock_stack[thread_id][filepath] += 1
try:
yield
finally:
# 减少重入计数
self.lock_stack[thread_id][filepath] -= 1
if self.lock_stack[thread_id][filepath] <= 0:
del self.lock_stack[thread_id][filepath]
if not self.lock_stack[thread_id]:
del self.lock_stack[thread_id]
return
# 确保文件存在
if not os.path.exists(filepath):
# 创建文件的父目录
os.makedirs(os.path.dirname(filepath) or '.', exist_ok=True)
with open(filepath, 'w') as f:
f.write('') # 创建空文件
file_obj = None
try:
# 打开文件
file_obj = open(filepath, 'r+')
# 设置锁类型
if lock_type == 'shared':
lock_flag = fcntl.LOCK_SH
elif lock_type == 'exclusive':
lock_flag = fcntl.LOCK_EX
else:
raise ValueError(f"不支持的锁类型: {lock_type}")
if not blocking:
lock_flag |= fcntl.LOCK_NB
# 尝试获取锁
start_time = time.time()
while True:
try:
fcntl.flock(file_obj.fileno(), lock_flag)
break
except BlockingIOError:
if not blocking or (time.time() - start_time) > self.timeout:
raise TimeoutError(f"获取文件锁超时: {filepath}")
time.sleep(0.1)
# 记录锁信息
self.locked_files[filepath] = (file_obj, lock_type)
# 初始化线程锁栈
if thread_id not in self.lock_stack:
self.lock_stack[thread_id] = {}
self.lock_stack[thread_id][filepath] = 1
yield
finally:
# 释放锁
if filepath in self.locked_files:
locked_file, _ = self.locked_files[filepath]
try:
fcntl.flock(locked_file.fileno(), fcntl.LOCK_UN)
locked_file.close()
except:
pass
del self.locked_files[filepath]
# 清理线程锁栈
if thread_id in self.lock_stack and filepath in self.lock_stack[thread_id]:
del self.lock_stack[thread_id][filepath]
if not self.lock_stack[thread_id]:
del self.lock_stack[thread_id]
def is_locked(self, filepath: str) -> bool:
"""检查文件是否被锁定"""
return filepath in self.locked_files
def get_lock_info(self, filepath: str) -> Optional[dict]:
"""获取文件锁信息"""
if filepath in self.locked_files:
file_obj, lock_type = self.locked_files[filepath]
return {
'filepath': filepath,
'lock_type': lock_type,
'file_descriptor': file_obj.fileno() if file_obj else None
}
return None
def release_all_locks(self):
"""释放所有锁"""
for filepath in list(self.locked_files.keys()):
file_obj, _ = self.locked_files[filepath]
try:
fcntl.flock(file_obj.fileno(), fcntl.LOCK_UN)
file_obj.close()
except:
pass
self.locked_files.clear()
self.lock_stack.clear()
def demo_basic_file_locking():
"""演示基础文件锁操作"""
print("=== 基础文件锁演示 ===")
lock_manager = FileLockManager(timeout=5.0)
test_file = 'test_lock_file.txt'
# 1. 基本锁操作
print("\n1. 基本锁操作:")
with lock_manager.acquire_lock(test_file, 'exclusive'):
print(f" 获得排他锁: {test_file}")
with open(test_file, 'w') as f:
f.write("这是在排他锁保护下写入的内容\n")
# 在锁内读取
with open(test_file, 'r') as f:
content = f.read()
print(f" 锁内读取内容: {repr(content)}")
print(" 排他锁已释放")
# 2. 共享锁测试
print("\n2. 共享锁测试:")
with lock_manager.acquire_lock(test_file, 'shared'):
print(f" 获得共享锁: {test_file}")
with open(test_file, 'r') as f:
content = f.read()
print(f" 共享锁读取: {repr(content)}")
print(" 共享锁已释放")
# 3. 锁信息查询
print("\n3. 锁信息:")
is_locked = lock_manager.is_locked(test_file)
print(f" 文件是否被锁定: {is_locked}")
demo_basic_file_locking()
---
02.多进程文件锁应用
a.跨进程锁挑战
a.进程隔离
不同进程有独立的内存空间，需要通过文件系统共享锁状态。
b.进程崩溃处理
进程崩溃时锁可能不会被正确释放。
c.锁继承
子进程需要正确处理从父进程继承的文件描述符。
b.解决方案
a.锁文件机制
使用专门的锁文件来协调进程间的访问。
b.心跳机制
定期更新锁文件的时间戳，检测活锁。
c.优雅降级
当锁获取失败时提供替代方案。
c.多进程锁实现
---
# 多进程文件锁实现
import multiprocessing
import signal
import atexit
from typing import Dict, List
class MultiProcessFileLock:
def __init__(self, lock_file_path: str, timeout: float = 30.0):
self.lock_file_path = lock_file_path
self.timeout = timeout
self.process_id = os.getpid()
self.lock_info = {
'pid': self.process_id,
'timestamp': time.time(),
'hostname': os.uname().nodename if hasattr(os, 'uname') else 'unknown'
}
self.is_holding = False
# 注册退出处理
atexit.register(self.force_release)
# 注册信号处理
signal.signal(signal.SIGTERM, self._signal_handler)
signal.signal(signal.SIGINT, self._signal_handler)
def _signal_handler(self, signum, frame):
"""信号处理器，确保锁被释放"""
self.force_release()
os._exit(0)
def _write_lock_info(self):
"""写入锁信息到锁文件"""
self.lock_info['timestamp'] = time.time()
lock_data = json.dumps(self.lock_info, ensure_ascii=False)
with open(self.lock_file_path, 'w', encoding='utf-8') as f:
f.write(lock_data)
def _read_lock_info(self) -> Optional[Dict]:
"""读取锁信息"""
try:
with open(self.lock_file_path, 'r', encoding='utf-8') as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return None
def _is_lock_valid(self) -> bool:
"""检查锁是否仍然有效"""
lock_info = self._read_lock_info()
if not lock_info:
return False
# 检查时间戳
if time.time() - lock_info['timestamp'] > self.timeout:
return False
# 检查进程是否存在
try:
os.kill(lock_info['pid'], 0)
except ProcessLookupError:
return False
except PermissionError:
# 进程存在但无权限发送信号，认为有效
pass
return True
def acquire(self, blocking: bool = True) -> bool:
"""获取锁"""
start_time = time.time()
while True:
try:
# 尝试创建锁文件（原子操作）
with open(self.lock_file_path, 'x') as f:
# 文件创建成功，获得锁
self._write_lock_info()
self.is_holding = True
return True
except FileExistsError:
# 锁文件已存在，检查是否有效
if not self._is_lock_valid():
# 锁无效，尝试删除并重新获取
try:
os.remove(self.lock_file_path)
continue
except OSError:
pass
if not blocking:
return False
# 检查超时
if time.time() - start_time > self.timeout:
return False
# 等待一段时间后重试
time.sleep(0.1)
def release(self) -> bool:
"""释放锁"""
if not self.is_holding:
return True
try:
# 验证锁文件内容
lock_info = self._read_lock_info()
if lock_info and lock_info['pid'] == self.process_id:
os.remove(self.lock_file_path)
self.is_holding = False
return True
except OSError:
return False
def force_release(self):
"""强制释放锁（用于异常退出）"""
if self.is_holding:
try:
if os.path.exists(self.lock_file_path):
os.remove(self.lock_file_path)
except:
pass
self.is_holding = False
def __enter__(self):
if not self.acquire():
raise RuntimeError(f"无法获取锁: {self.lock_file_path}")
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.release()
class SafeFileWriter:
"""安全的多进程文件写入器"""
def __init__(self, target_file: str, lock_dir: str = '.locks'):
self.target_file = target_file
self.lock_dir = lock_dir
os.makedirs(lock_dir, exist_ok=True)
# 为目标文件创建对应的锁文件路径
lock_filename = f".{os.path.basename(target_file)}.lock"
self.lock_path = os.path.join(lock_dir, lock_filename)
def safe_write(self, content: str, process_name: str = "unknown") -> bool:
"""安全写入文件"""
lock = MultiProcessFileLock(self.lock_path, timeout=10.0)
try:
with lock:
# 获得锁后进行写入操作
temp_file = self.target_file + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
f.write(f"# Written by {process_name} (PID: {os.getpid()})\n")
f.write(f"# Timestamp: {time.time()}\n")
f.write(content)
# 原子重命名
if os.path.exists(self.target_file):
backup_file = self.target_file + '.bak'
if os.path.exists(backup_file):
os.remove(backup_file)
os.rename(self.target_file, backup_file)
os.rename(temp_file, self.target_file)
return True
except Exception as e:
print(f"进程 {process_name} 写入失败: {e}")
return False
def safe_read(self) -> Optional[str]:
"""安全读取文件"""
lock = MultiProcessFileLock(self.lock_path, timeout=5.0)
try:
with lock:
if os.path.exists(self.target_file):
with open(self.target_file, 'r', encoding='utf-8') as f:
return f.read()
return None
except Exception as e:
print(f"读取文件失败: {e}")
return None
def worker_process(process_id: int, target_file: str, iterations: int):
"""工作进程函数"""
writer = SafeFileWriter(target_file)
process_name = f"Worker-{process_id}"
for i in range(iterations):
content = f"Content from {process_name}, iteration {i}\n"
success = writer.safe_write(content, process_name)
if success:
print(f"{process_name}: 写入成功 (迭代 {i+1}/{iterations})")
else:
print(f"{process_name}: 写入失败 (迭代 {i+1}/{iterations})")
# 随机等待，模拟实际工作
time.sleep(0.1 + (process_id % 3) * 0.05)
def demo_multiprocess_file_locking():
"""演示多进程文件锁"""
print("=== 多进程文件锁演示 ===")
target_file = 'shared_output.txt'
# 清理旧文件
for ext in ['', '.tmp', '.bak']:
file_path = target_file + ext
if os.path.exists(file_path):
os.remove(file_path)
# 创建多个工作进程
num_processes = 4
iterations_per_process = 3
processes = []
print(f"启动 {num_processes} 个进程，每个进程写入 {iterations_per_process} 次")
start_time = time.time()
for i in range(num_processes):
process = multiprocessing.Process(
target=worker_process,
args=(i, target_file, iterations_per_process)
)
processes.append(process)
process.start()
# 等待所有进程完成
for process in processes:
process.join()
elapsed_time = time.time() - start_time
print(f"\n所有进程完成，总耗时: {elapsed_time:.2f}秒")
# 检查最终结果
if os.path.exists(target_file):
with open(target_file, 'r', encoding='utf-8') as f:
content = f.read()
lines = content.strip().split('\n')
print(f"\n最终文件内容 ({len(lines)} 行):")
for line in lines[:5]: # 只显示前5行
print(f" {line}")
if len(lines) > 5:
print(f" ... (还有 {len(lines) - 5} 行)")
else:
print("\n目标文件不存在")
if __name__ == "__main__":
demo_multiprocess_file_locking()
---
03.高级文件锁模式
a.读写锁实现
a.多读单写
允许多个读者同时访问，但写者需要独占访问。
b.锁升级
允许读锁升级为写锁，但需要处理竞争条件。
c.优先级处理
处理读写请求的优先级，避免写者饥饿。
b.分布式文件锁
a.网络文件系统
处理NFS等网络文件系统的锁特性。
b.租约机制
使用租约机制处理网络分区和进程崩溃。
c.共识算法
使用分布式共识算法保证锁的一致性。
c.性能优化
a.锁池化
重用锁对象减少创建开销。
b.批量操作
支持批量获取多个文件的锁。
c.异步锁
支持异步获取和释放锁。
d.高级锁实现
---
# 高级文件锁实现
import asyncio
import weakref
from enum import Enum
from collections import defaultdict
from dataclasses import dataclass
class LockType(Enum):
SHARED = "shared"
EXCLUSIVE = "exclusive"
@dataclass
class LockRequest:
process_id: int
lock_type: LockType
timestamp: float
future: asyncio.Future
class AdvancedFileLockManager:
def __init__(self, lock_dir: str = '.advanced_locks'):
self.lock_dir = lock_dir
os.makedirs(lock_dir, exist_ok=True)
# 锁状态管理
self.file_locks = {} # 文件路径 -> 锁信息
self.waiting_queue = defaultdict(list) # 文件路径 -> 等待队列
# 性能优化
self.lock_pool = weakref.WeakValueDictionary()
self.batch_operations = []
# 异步支持
self.loop = asyncio.new_event_loop()
self.executor = None
def _get_lock_path(self, filepath: str) -> str:
"""获取锁文件路径"""
safe_name = filepath.replace('/', '_').replace('\\', '_')
return os.path.join(self.lock_dir, f"{safe_name}.adv.lock")
def _can_grant_lock(self, filepath: str, lock_type: LockType) -> bool:
"""检查是否可以授予锁"""
if filepath not in self.file_locks:
return True
current_locks = self.file_locks[filepath]
if lock_type == LockType.SHARED:
# 共享锁：只有当没有排他锁时才能授予
return all(lock['type'] == LockType.SHARED for lock in current_locks)
else:
# 排他锁：只有在没有任何锁时才能授予
return len(current_locks) == 0
def _grant_lock(self, filepath: str, request: LockRequest):
"""授予锁"""
if filepath not in self.file_locks:
self.file_locks[filepath] = []
lock_info = {
'process_id': request.process_id,
'type': request.lock_type,
'timestamp': request.timestamp
}
self.file_locks[filepath].append(lock_info)
# 更新锁文件
self._update_lock_file(filepath)
# 完成请求
if not request.future.done():
request.future.set_result(True)
def _release_lock(self, filepath: str, process_id: int):
"""释放锁"""
if filepath not in self.file_locks:
return
# 移除该进程的所有锁
self.file_locks[filepath] = [
lock for lock in self.file_locks[filepath]
if lock['process_id'] != process_id
]
if not self.file_locks[filepath]:
del self.file_locks[filepath]
# 更新锁文件
self._update_lock_file(filepath)
# 处理等待队列
self._process_waiting_queue(filepath)
def _update_lock_file(self, filepath: str):
"""更新锁文件"""
lock_path = self._get_lock_path(filepath)
if filepath not in self.file_locks or not self.file_locks[filepath]:
# 没有锁，删除锁文件
if os.path.exists(lock_path):
os.remove(lock_path)
return
# 写入锁信息
lock_data = {
'filepath': filepath,
'locks': self.file_locks[filepath],
'timestamp': time.time()
}
temp_file = lock_path + '.tmp'
with open(temp_file, 'w', encoding='utf-8') as f:
json.dump(lock_data, f, indent=2, ensure_ascii=False)
os.rename(temp_file, lock_path)
def _process_waiting_queue(self, filepath: str):
"""处理等待队列"""
if filepath not in self.waiting_queue:
return
# 按时间戳排序等待请求
waiting = self.waiting_queue[filepath]
waiting.sort(key=lambda x: x.timestamp)
granted_requests = []
for request in waiting:
if self._can_grant_lock(filepath, request.lock_type):
self._grant_lock(filepath, request)
granted_requests.append(request)
# 移除已授予的请求
for request in granted_requests:
waiting.remove(request)
if not waiting:
del self.waiting_queue[filepath]
async def acquire_lock_async(self, filepath: str, lock_type: LockType,
timeout: float = 30.0) -> bool:
"""异步获取锁"""
request = LockRequest(
process_id=os.getpid(),
lock_type=lock_type,
timestamp=time.time(),
future=self.loop.create_future()
)
# 检查是否可以立即授予锁
if self._can_grant_lock(filepath, lock_type):
self._grant_lock(filepath, request)
return await request.future
# 添加到等待队列
self.waiting_queue[filepath].append(request)
try:
# 等待锁授予或超时
await asyncio.wait_for(request.future, timeout=timeout)
return True
except asyncio.TimeoutError:
# 超时，从等待队列中移除
self.waiting_queue[filepath].remove(request)
if not self.waiting_queue[filepath]:
del self.waiting_queue[filepath]
return False
def acquire_lock(self, filepath: str, lock_type: LockType,
timeout: float = 30.0) -> bool:
"""同步获取锁"""
if not self.loop.is_running():
asyncio.set_event_loop(self.loop)
return self.loop.run_until_complete(
self.acquire_lock_async(filepath, lock_type, timeout)
)
def release_lock(self, filepath: str) -> bool:
"""释放锁"""
self._release_lock(filepath, os.getpid())
return True
async def acquire_batch_locks(self, file_lock_pairs: List[tuple],
timeout: float = 30.0) -> bool:
"""批量获取锁"""
# 按文件名排序以避免死锁
sorted_pairs = sorted(file_lock_pairs, key=lambda x: x[0])
acquired_locks = []
try:
for filepath, lock_type in sorted_pairs:
success = await self.acquire_lock_async(filepath, lock_type, timeout)
if success:
acquired_locks.append(filepath)
else:
# 获取失败，释放已获取的锁
for acquired_path in acquired_locks:
self.release_lock(acquired_path)
return False
return True
except Exception:
# 异常时释放所有已获取的锁
for acquired_path in acquired_locks:
self.release_lock(acquired_path)
raise
def get_lock_status(self, filepath: str) -> Optional[dict]:
"""获取文件锁状态"""
if filepath not in self.file_locks:
return None
return {
'filepath': filepath,
'locks': self.file_locks[filepath],
'waiting_count': len(self.waiting_queue.get(filepath, []))
}
def get_all_locks(self) -> dict:
"""获取所有锁的状态"""
status = {}
for filepath in self.file_locks:
status[filepath] = self.get_lock_status(filepath)
return status
def cleanup_stale_locks(self, timeout: float = 300.0):
"""清理超时锁"""
current_time = time.time()
stale_files = []
for filepath, locks in self.file_locks.items():
# 检查是否有超时的锁
stale_locks = [
lock for lock in locks
if current_time - lock['timestamp'] > timeout
]
if stale_locks:
# 检查进程是否还存在
for lock in stale_locks:
try:
os.kill(lock['process_id'], 0)
except ProcessLookupError:
# 进程不存在，清理锁
self._release_lock(filepath, lock['process_id'])
# 更新状态
if not self.file_locks[filepath]:
stale_files.append(filepath)
# 清理没有锁的文件
for filepath in stale_files:
if filepath in self.file_locks:
del self.file_locks[filepath]
return len(stale_files)
def demo_advanced_file_locking():
"""演示高级文件锁功能"""
print("=== 高级文件锁演示 ===")
lock_manager = AdvancedFileLockManager('demo_advanced_locks')
# 1. 基本锁操作
print("\n1. 基本锁操作:")
files = ['file1.txt', 'file2.txt', 'file3.txt']
# 获取共享锁
for file in files[:2]:
success = lock_manager.acquire_lock(file, LockType.SHARED)
print(f" 获取共享锁 {file}: {'成功' if success else '失败'}")
# 尝试获取排他锁（应该失败）
success = lock_manager.acquire_lock(files[0], LockType.EXCLUSIVE, timeout=2.0)
print(f" 获取排他锁 {files[0]}: {'成功' if success else '失败'}")
# 显示锁状态
print("\n 当前锁状态:")
for file in files:
status = lock_manager.get_lock_status(file)
if status:
print(f" {file}: {len(status['locks'])} 个锁, {status['waiting_count']} 个等待")
# 2. 批量锁操作
print("\n2. 批量锁操作:")
batch_files = [('batch1.txt', LockType.EXCLUSIVE), ('batch2.txt', LockType.SHARED)]
success = lock_manager.acquire_lock(batch_files[0][0], batch_files[0][1])
print(f" 批量获取锁: {'成功' if success else '失败'}")
# 3. 清理超时锁
print("\n3. 清理超时锁:")
cleaned = lock_manager.cleanup_stale_locks(timeout=1.0)
print(f" 清理了 {cleaned} 个超时锁")
# 释放所有锁
for file in files:
lock_manager.release_lock(file)
demo_advanced_file_locking()
---
8.5 错误恢复
01.错误类型分析
a.系统级错误
a.磁盘空间不足
写入文件时遇到存储空间耗尽的情况。
b.权限错误
文件权限不足导致无法访问或修改。
c.文件系统损坏
底层文件系统出现错误或损坏。
b.应用级错误
a.数据格式错误
文件内容格式不符合预期。
b.数据完整性错误
文件内容不完整或被意外截断。
c.并发冲突
多个进程同时操作导致的冲突。
c.网络相关错误
a.网络中断
网络连接中断导致的操作失败。
b.远程存储错误
网络存储设备出现故障。
c.超时错误
网络操作超时导致的失败。
d.错误处理基础实现
---
# 错误处理与恢复基础实现
import os
import time
import json
import logging
import traceback
from datetime import datetime, timedelta
from typing import Dict, List, Any, Optional, Callable
class FileOperationError(Exception):
"""文件操作错误基类"""
def __init__(self, message: str, error_code: str = None, details: Dict = None):
super().__init__(message)
self.error_code = error_code
self.details = details or {}
self.timestamp = time.time()
class FileErrorHandler:
def __init__(self, log_file: str = 'file_operations.log'):
self.log_file = log_file
self.error_history = []
self.recovery_strategies = {}
self.max_history = 1000
# 设置日志
self._setup_logging()
def _setup_logging(self):
"""设置日志记录"""
self.logger = logging.getLogger('FileErrorHandler')
self.logger.setLevel(logging.DEBUG)
# 文件处理器
file_handler = logging.FileHandler(self.log_file, encoding='utf-8')
file_handler.setLevel(logging.DEBUG)
# 控制台处理器
console_handler = logging.StreamHandler()
console_handler.setLevel(logging.WARNING)
# 格式化器
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s'
)
file_handler.setFormatter(formatter)
console_handler.setFormatter(formatter)
self.logger.addHandler(file_handler)
self.logger.addHandler(console_handler)
def register_recovery_strategy(self, error_code: str, strategy: Callable):
"""注册恢复策略"""
self.recovery_strategies[error_code] = strategy
def handle_error(self, error: Exception, context: Dict = None) -> bool:
"""处理错误"""
error_info = {
'type': type(error).__name__,
'message': str(error),
'timestamp': datetime.now().isoformat(),
'context': context or {},
'traceback': traceback.format_exc()
}
# 记录错误
self.logger.error(f"文件操作错误: {error_info}")
# 添加到历史记录
self.error_history.append(error_info)
if len(self.error_history) > self.max_history:
self.error_history.pop(0)
# 尝试恢复
return self._attempt_recovery(error, error_info)
def _attempt_recovery(self, error: Exception, error_info: Dict) -> bool:
"""尝试错误恢复"""
error_code = None
# 确定错误类型
if isinstance(error, PermissionError):
error_code = 'PERMISSION_DENIED'
elif isinstance(error, FileNotFoundError):
error_code = 'FILE_NOT_FOUND'
elif isinstance(error, OSError):
if 'No space left' in str(error):
error_code = 'DISK_FULL'
elif 'Input/output error' in str(error):
error_code = 'IO_ERROR'
elif 'Device or resource busy' in str(error):
error_code = 'RESOURCE_BUSY'
elif isinstance(error, json.JSONDecodeError):
error_code = 'JSON_FORMAT_ERROR'
# 执行恢复策略
if error_code and error_code in self.recovery_strategies:
try:
strategy = self.recovery_strategies[error_code]
return strategy(error, error_info)
except Exception as recovery_error:
self.logger.error(f"恢复策略执行失败: {recovery_error}")
return False
# 默认恢复策略
return self._default_recovery(error, error_info)
def _default_recovery(self, error: Exception, error_info: Dict) -> bool:
"""默认恢复策略"""
if isinstance(error, PermissionError):
return self._recover_permission_error(error, error_info)
elif isinstance(error, FileNotFoundError):
return self._recover_file_not_found(error, error_info)
elif isinstance(error, OSError):
return self._recover_os_error(error, error_info)
return False
def _recover_permission_error(self, error: Exception, error_info: Dict) -> bool:
"""恢复权限错误"""
filepath = error_info.get('context', {}).get('filepath')
if not filepath:
return False
try:
# 尝试修复权限
parent_dir = os.path.dirname(filepath)
if parent_dir and not os.access(parent_dir, os.W_OK):
# 父目录不可写，尝试创建临时目录
temp_dir = os.path.join(os.path.dirname(parent_dir), 'temp_write')
os.makedirs(temp_dir, exist_ok=True)
return True
# 尝试修改文件权限
if os.path.exists(filepath):
os.chmod(filepath, 0o644)
else:
# 创建文件的父目录
os.makedirs(os.path.dirname(filepath), exist_ok=True)
return True
except Exception:
return False
def _recover_file_not_found(self, error: Exception, error_info: Dict) -> bool:
"""恢复文件不存在错误"""
filepath = error_info.get('context', {}).get('filepath')
if not filepath:
return False
try:
# 创建文件和必要的目录
parent_dir = os.path.dirname(filepath)
if parent_dir:
os.makedirs(parent_dir, exist_ok=True)
with open(filepath, 'w') as f:
f.write('') # 创建空文件
return True
except Exception:
return False
def _recover_os_error(self, error: Exception, error_info: Dict) -> bool:
"""恢复操作系统错误"""
error_msg = str(error)
if 'No space left' in error_msg:
return self._recover_disk_full(error_info)
elif 'Device or resource busy' in error_msg:
return self._recover_resource_busy(error_info)
elif 'Input/output error' in error_msg:
return self._recover_io_error(error_info)
return False
def _recover_disk_full(self, error_info: Dict) -> bool:
"""恢复磁盘空间不足错误"""
try:
# 清理临时文件
import tempfile
temp_dir = tempfile.gettempdir()
for filename in os.listdir(temp_dir):
if filename.startswith('tmp'):
filepath = os.path.join(temp_dir, filename)
try:
# 只删除超过1小时的临时文件
if time.time() - os.path.getctime(filepath) > 3600:
os.remove(filepath)
except:
pass
return True
except Exception:
return False
def _recover_resource_busy(self, error_info: Dict) -> bool:
"""恢复资源忙错误"""
filepath = error_info.get('context', {}).get('filepath')
if not filepath:
return False
try:
# 等待资源释放
max_wait = 30
for i in range(max_wait):
try:
# 尝试访问文件
with open(filepath, 'a') as f:
pass
return True
except:
time.sleep(1)
return False
except Exception:
return False
def _recover_io_error(self, error_info: Dict) -> bool:
"""恢复IO错误"""
filepath = error_info.get('context', {}).get('filepath')
if not filepath:
return False
try:
# 尝试重新挂载文件系统或等待恢复
time.sleep(5)
# 检查文件是否可访问
return os.access(filepath, os.R_OK | os.W_OK)
except Exception:
return False
def get_error_statistics(self) -> Dict:
"""获取错误统计信息"""
if not self.error_history:
return {'total_errors': 0}
error_types = {}
recent_errors = [
err for err in self.error_history
if time.time() - time.mktime(
datetime.fromisoformat(err['timestamp']).timetuple()
) < 3600 # 最近1小时
]
for error in self.error_history:
error_type = error['type']
error_types[error_type] = error_types.get(error_type, 0) + 1
return {
'total_errors': len(self.error_history),
'recent_errors': len(recent_errors),
'error_types': error_types,
'last_error': self.error_history[-1] if self.error_history else None
}
def demo_error_handling():
"""演示错误处理"""
print("=== 错误处理与恢复演示 ===")
error_handler = FileErrorHandler('demo_errors.log')
# 注册自定义恢复策略
def custom_json_recovery(error, error_info):
print(" 执行JSON格式错误恢复策略")
filepath = error_info.get('context', {}).get('filepath')
if filepath and os.path.exists(filepath):
# 尝试修复JSON文件
with open(filepath, 'r') as f:
content = f.read()
# 简单的JSON修复：移除末尾的逗号
fixed_content = content.rstrip().rstrip(',')
if fixed_content != content:
with open(filepath, 'w') as f:
f.write(fixed_content)
return True
return False
error_handler.register_recovery_strategy('JSON_FORMAT_ERROR', custom_json_recovery)
# 测试不同类型的错误
test_cases = [
{
'name': '权限错误',
'error': PermissionError("Permission denied"),
'context': {'filepath': '/root/test.txt', 'operation': 'write'}
},
{
'name': '文件不存在错误',
'error': FileNotFoundError("No such file or directory"),
'context': {'filepath': 'nonexistent.txt', 'operation': 'read'}
},
{
'name': 'JSON格式错误',
'error': json.JSONDecodeError("Expecting ',' delimiter", 'test.json', 10),
'context': {'filepath': 'test.json', 'operation': 'parse'}
}
]
for test_case in test_cases:
print(f"\n测试: {test_case['name']}")
success = error_handler.handle_error(
test_case['error'],
test_case['context']
)
print(f" 恢复结果: {'成功' if success else '失败'}")
# 显示错误统计
print("\n错误统计:")
stats = error_handler.get_error_statistics()
for key, value in stats.items():
print(f" {key}: {value}")
demo_error_handling()
---
02.事务性文件操作恢复
a.事务状态管理
a.开始状态
操作开始，但尚未修改任何文件。
b.进行中状态
部分操作已完成，但事务尚未提交。
c.提交状态
所有操作完成，事务已提交。
d.回滚状态
操作失败，正在回滚已完成的操作。
b.恢复策略
a.前向恢复
从检查点开始继续执行未完成的操作。
b.后向恢复
撤销已执行的操作，回到一致状态。
c.补偿事务
执行相反的操作来抵消已完成操作的影响。
c.事务性恢复实现
---
# 事务性文件操作恢复实现
import sqlite3
import threading
from contextlib import contextmanager
from dataclasses import dataclass
from enum import Enum
class TransactionState(Enum):
PENDING = "pending"
ACTIVE = "active"
COMMITTED = "committed"
ROLLED_BACK = "rolled_back"
RECOVERING = "recovering"
class OperationType(Enum):
CREATE = "create"
WRITE = "write"
DELETE = "delete"
RENAME = "rename"
COPY = "copy"
@dataclass
class FileOperation:
operation_type: OperationType
source_path: str
target_path: Optional[str] = None
backup_path: Optional[str] = None
data: Optional[bytes] = None
timestamp: float = None
def __post_init__(self):
if self.timestamp is None:
self.timestamp = time.time()
class TransactionalFileManager:
def __init__(self, db_path: str = 'transaction_log.db'):
self.db_path = db_path
self.current_transaction = None
self.transaction_lock = threading.RLock()
# 初始化事务日志数据库
self._init_database()
def _init_database(self):
"""初始化事务日志数据库"""
with sqlite3.connect(self.db_path) as conn:
conn.execute('''
CREATE TABLE IF NOT EXISTS transactions (
transaction_id TEXT PRIMARY KEY,
state TEXT NOT NULL,
created_at REAL NOT NULL,
updated_at REAL NOT NULL,
operations_count INTEGER DEFAULT 0
)
''')
conn.execute('''
CREATE TABLE IF NOT EXISTS operations (
operation_id INTEGER PRIMARY KEY AUTOINCREMENT,
transaction_id TEXT NOT NULL,
operation_type TEXT NOT NULL,
source_path TEXT NOT NULL,
target_path TEXT,
backup_path TEXT,
data BLOB,
timestamp REAL NOT NULL,
executed BOOLEAN DEFAULT FALSE,
FOREIGN KEY (transaction_id) REFERENCES transactions (transaction_id)
)
''')
conn.execute('''
CREATE INDEX IF NOT EXISTS idx_transaction_operations
ON operations (transaction_id)
''')
@contextmanager
def begin_transaction(self):
"""开始事务"""
transaction_id = self._create_transaction()
self.current_transaction = transaction_id
try:
yield self
self.commit_transaction()
except Exception as e:
self.rollback_transaction()
raise e
finally:
self.current_transaction = None
def _create_transaction(self) -> str:
"""创建新事务"""
transaction_id = f"tx_{int(time.time() * 1000000)}_{os.getpid()}"
with sqlite3.connect(self.db_path) as conn:
conn.execute('''
INSERT INTO transactions (transaction_id, state, created_at, updated_at)
VALUES (?, ?, ?, ?)
''', (transaction_id, TransactionState.ACTIVE.value, time.time(), time.time()))
return transaction_id
def _update_transaction_state(self, state: TransactionState):
"""更新事务状态"""
if not self.current_transaction:
return
with sqlite3.connect(self.db_path) as conn:
conn.execute('''
UPDATE transactions
SET state = ?, updated_at = ?
WHERE transaction_id = ?
''', (state.value, time.time(), self.current_transaction))
def _log_operation(self, operation: FileOperation):
"""记录操作到事务日志"""
if not self.current_transaction:
raise ValueError("没有活跃的事务")
with sqlite3.connect(self.db_path) as conn:
conn.execute('''
INSERT INTO operations
(transaction_id, operation_type, source_path, target_path, backup_path, data, timestamp)
VALUES (?, ?, ?, ?, ?, ?, ?)
''', (
self.current_transaction,
operation.operation_type.value,
operation.source_path,
operation.target_path,
operation.backup_path,
operation.data,
operation.timestamp
))
def execute_operation(self, operation: FileOperation) -> bool:
"""执行文件操作"""
if not self.current_transaction:
raise ValueError("没有活跃的事务")
try:
success = self._perform_file_operation(operation)
if success:
# 标记操作为已执行
self._mark_operation_executed(operation)
return success
except Exception as e:
print(f"操作执行失败: {e}")
return False
def _perform_file_operation(self, operation: FileOperation) -> bool:
"""执行实际的文件操作"""
try:
if operation.operation_type == OperationType.CREATE:
return self._create_file(operation)
elif operation.operation_type == OperationType.WRITE:
return self._write_file(operation)
elif operation.operation_type == OperationType.DELETE:
return self._delete_file(operation)
elif operation.operation_type == OperationType.RENAME:
return self._rename_file(operation)
elif operation.operation_type == OperationType.COPY:
return self._copy_file(operation)
else:
raise ValueError(f"不支持的操作类型: {operation.operation_type}")
except Exception as e:
print(f"文件操作失败: {e}")
return False
def _create_file(self, operation: FileOperation) -> bool:
"""创建文件"""
# 确保目录存在
os.makedirs(os.path.dirname(operation.source_path) or '.', exist_ok=True)
with open(operation.source_path, 'wb') as f:
if operation.data:
f.write(operation.data)
return True
def _write_file(self, operation: FileOperation) -> bool:
"""写入文件"""
# 创建备份
if os.path.exists(operation.source_path):
backup_path = f"{operation.source_path}.bak.{int(time.time())}"
import shutil
shutil.copy2(operation.source_path, backup_path)
operation.backup_path = backup_path
# 写入新内容
with open(operation.source_path, 'wb') as f:
f.write(operation.data or b'')
return True
def _delete_file(self, operation: FileOperation) -> bool:
"""删除文件"""
# 创建备份（移动到回收站目录）
if os.path.exists(operation.source_path):
trash_dir = os.path.join(os.path.dirname(operation.source_path), '.trash')
os.makedirs(trash_dir, exist_ok=True)
backup_name = f"{int(time.time())}_{os.path.basename(operation.source_path)}"
backup_path = os.path.join(trash_dir, backup_name)
shutil.move(operation.source_path, backup_path)
operation.backup_path = backup_path
return True
def _rename_file(self, operation: FileOperation) -> bool:
"""重命名文件"""
if os.path.exists(operation.source_path):
if os.path.exists(operation.target_path):
# 目标文件存在，创建备份
backup_path = f"{operation.target_path}.bak.{int(time.time())}"
shutil.copy2(operation.target_path, backup_path)
# 可以将备份路径记录到某个地方
os.rename(operation.source_path, operation.target_path)
return True
return False
def _copy_file(self, operation: FileOperation) -> bool:
"""复制文件"""
if os.path.exists(operation.source_path):
os.makedirs(os.path.dirname(operation.target_path) or '.', exist_ok=True)
shutil.copy2(operation.source_path, operation.target_path)
return True
return False
def _mark_operation_executed(self, operation: FileOperation):
"""标记操作为已执行"""
with sqlite3.connect(self.db_path) as conn:
conn.execute('''
UPDATE operations
SET executed = TRUE
WHERE transaction_id = ?
AND source_path = ?
AND timestamp = ?
''', (self.current_transaction, operation.source_path, operation.timestamp))
def commit_transaction(self):
"""提交事务"""
if not self.current_transaction:
return
try:
self._update_transaction_state(TransactionState.COMMITTED)
print(f"事务 {self.current_transaction} 已提交")
except Exception as e:
print(f"提交事务失败: {e}")
self.rollback_transaction()
raise
def rollback_transaction(self):
"""回滚事务"""
if not self.current_transaction:
return
self._update_transaction_state(TransactionState.RECOVERING)
try:
# 获取所有操作，按时间戳倒序排列
operations = self._get_transaction_operations(self.current_transaction)
operations.reverse() # 撤销操作需要倒序执行
for operation in operations:
if operation.executed:
self._rollback_operation(operation)
self._update_transaction_state(TransactionState.ROLLED_BACK)
print(f"事务 {self.current_transaction} 已回滚")
except Exception as e:
print(f"回滚事务失败: {e}")
raise
def _rollback_operation(self, operation: FileOperation):
"""撤销单个操作"""
try:
if operation.operation_type == OperationType.CREATE:
# 删除创建的文件
if os.path.exists(operation.source_path):
os.remove(operation.source_path)
elif operation.operation_type == OperationType.WRITE:
# 恢复备份文件
if operation.backup_path and os.path.exists(operation.backup_path):
shutil.move(operation.backup_path, operation.source_path)
elif operation.operation_type == OperationType.DELETE:
# 从回收站恢复文件
if operation.backup_path and os.path.exists(operation.backup_path):
os.makedirs(os.path.dirname(operation.source_path) or '.', exist_ok=True)
shutil.move(operation.backup_path, operation.source_path)
elif operation.operation_type == OperationType.RENAME:
# 撤销重命名
if os.path.exists(operation.target_path):
os.rename(operation.target_path, operation.source_path)
except Exception as e:
print(f"撤销操作失败: {e}")
def _get_transaction_operations(self, transaction_id: str) -> List[FileOperation]:
"""获取事务的所有操作"""
operations = []
with sqlite3.connect(self.db_path) as conn:
cursor = conn.execute('''
SELECT operation_type, source_path, target_path, backup_path, data, timestamp, executed
FROM operations
WHERE transaction_id = ?
ORDER BY timestamp ASC
''', (transaction_id,))
for row in cursor:
operation = FileOperation(
operation_type=OperationType(row[0]),
source_path=row[1],
target_path=row[2],
backup_path=row[3],
data=row[4],
timestamp=row[5]
)
operations.append(operation)
return operations
def recover_pending_transactions(self):
"""恢复未完成的事务"""
with sqlite3.connect(self.db_path) as conn:
cursor = conn.execute('''
SELECT transaction_id
FROM transactions
WHERE state = ?
''', (TransactionState.ACTIVE.value,))
pending_transactions = [row[0] for row in cursor]
for transaction_id in pending_transactions:
print(f"恢复未完成的事务: {transaction_id}")
self.current_transaction = transaction_id
try:
self.rollback_transaction()
except Exception as e:
print(f"恢复事务失败: {e}")
finally:
self.current_transaction = None
def demo_transactional_file_operations():
"""演示事务性文件操作"""
print("=== 事务性文件操作演示 ===")
manager = TransactionalFileManager('demo_transactions.db')
# 恢复未完成的事务
manager.recover_pending_transactions()
# 1. 正常事务
print("\n1. 正常事务操作:")
try:
with manager.begin_transaction():
# 创建文件
create_op = FileOperation(
operation_type=OperationType.CREATE,
source_path='test_file1.txt',
data=b'Hello, World!'
)
manager.execute_operation(create_op)
# 写入文件
write_op = FileOperation(
operation_type=OperationType.WRITE,
source_path='test_file2.txt',
data=b'This is a test file.'
)
manager.execute_operation(write_op)
print(" 事务操作成功完成")
except Exception as e:
print(f" 事务失败: {e}")
# 2. 失败事务（触发回滚）
print("\n2. 失败事务操作:")
try:
with manager.begin_transaction():
# 创建文件
create_op = FileOperation(
operation_type=OperationType.CREATE,
source_path='test_file3.txt',
data=b'This file should be rolled back'
)
manager.execute_operation(create_op)
# 检查文件是否创建
file_exists = os.path.exists('test_file3.txt')
print(f" 创建的文件存在: {file_exists}")
# 故意引发异常
raise ValueError("模拟事务失败")
except ValueError as e:
print(f" 事务失败并回滚: {e}")
# 检查回滚结果
file_exists_after_rollback = os.path.exists('test_file3.txt')
print(f" 回滚后文件存在: {file_exists_after_rollback}")
# 3. 模拟崩溃恢复
print("\n3. 模拟崩溃恢复:")
try:
with manager.begin_transaction():
# 创建一些文件
for i in range(3):
create_op = FileOperation(
operation_type=OperationType.CREATE,
source_path=f'crash_test_{i}.txt',
data=f'Content {i}'.encode()
)
manager.execute_operation(create_op)
# 模拟崩溃（不提交事务）
print(" 模拟系统崩溃...")
# 在实际应用中，这里进程会崩溃
# 为了演示，我们手动设置事务状态为ACTIVE
manager._update_transaction_state(TransactionState.ACTIVE)
except:
pass # 忽略异常
# 创建新的管理器实例来模拟重启
new_manager = TransactionalFileManager('demo_transactions.db')
new_manager.recover_pending_transactions()
# 检查恢复结果
crash_files_exist = [
os.path.exists(f'crash_test_{i}.txt') for i in range(3)
]
print(f" 崩溃后文件存在状态: {crash_files_exist}")
demo_transactional_file_operations()
---
03.自动化恢复系统
a.监控机制
a.健康检查
定期检查文件系统状态和文件完整性。
b.错误监控
实时监控系统错误和异常情况。
c.性能监控
监控文件操作的性能指标。
b.恢复策略
a.自动修复
对于常见错误自动执行修复策略。
b.渐进恢复
逐步恢复系统，避免过激操作。
c.人工干预
对于复杂情况提供人工干预接口。
c.预防措施
a.定期备份
自动创建重要文件的备份。
b.版本管理
维护文件的多版本历史。
c.冗余存储
使用多重存储提高可靠性。
d.自动化恢复实现
---
# 自动化恢复系统实现
import threading
import schedule
import queue
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timedelta
from enum import Enum
class RecoveryAction(Enum):
REPAIR_PERMISSIONS = "repair_permissions"
RECREATE_FILE = "recreate_file"
RESTORE_BACKUP = "restore_backup"
CLEANUP_TEMP = "cleanup_temp"
VERIFY_INTEGRITY = "verify_integrity"
@dataclass
class RecoveryTask:
file_path: str
action: RecoveryAction
priority: int = 0
max_retries: int = 3
retry_count: int = 0
created_at: float = None
def __post_init__(self):
if self.created_at is None:
self.created_at = time.time()
class AutomatedRecoverySystem:
def __init__(self, config_path: str = 'recovery_config.json'):
self.config_path = config_path
self.running = False
self.recovery_queue = queue.PriorityQueue()
self.executor = ThreadPoolExecutor(max_workers=4)
self.stats = {
'total_tasks': 0,
'successful_recoveries': 0,
'failed_recoveries': 0,
'active_tasks': 0
}
# 加载配置
self.config = self._load_config()
# 启动恢复工作线程
self.start_recovery_workers()
def _load_config(self) -> Dict:
"""加载恢复配置"""
default_config = {
'monitoring_interval': 300, # 5分钟
'backup_retention_days': 30,
'auto_recovery_enabled': True,
'max_recovery_attempts': 3,
'critical_paths': [
'/etc/config',
'/var/log',
'/data/important'
],
'recovery_strategies': {
'permission_errors': {
'enabled': True,
'strategy': 'repair_permissions'
},
'missing_files': {
'enabled': True,
'strategy': 'restore_backup'
},
'corrupted_files': {
'enabled': True,
'strategy': 'restore_from_version_control'
}
}
}
try:
if os.path.exists(self.config_path):
with open(self.config_path, 'r', encoding='utf-8') as f:
loaded_config = json.load(f)
# 合并配置
return {**default_config, **loaded_config}
except Exception as e:
print(f"加载配置失败，使用默认配置: {e}")
return default_config
def start_recovery_workers(self):
"""启动恢复工作线程"""
self.running = True
# 启动恢复任务处理器
for i in range(2):
threading.Thread(
target=self._recovery_worker,
name=f"RecoveryWorker-{i}",
daemon=True
).start()
# 启动监控线程
threading.Thread(
target=self._monitoring_worker,
name="MonitoringWorker",
daemon=True
).start()
# 启动统计报告线程
threading.Thread(
target=self._statistics_worker,
name="StatisticsWorker",
daemon=True
).start()
def _recovery_worker(self):
"""恢复工作线程"""
while self.running:
try:
# 获取恢复任务（优先级队列）
_, task = self.recovery_queue.get(timeout=5.0)
self.stats['active_tasks'] += 1
try:
success = self._execute_recovery_task(task)
if success:
self.stats['successful_recoveries'] += 1
else:
self.stats['failed_recoveries'] += 1
# 重新排队任务（如果还有重试机会）
if task.retry_count < task.max_retries:
task.retry_count += 1
task.priority += 1 # 降低优先级
self.recovery_queue.put((-task.priority, task))
except Exception as e:
print(f"恢复任务执行失败: {e}")
self.stats['failed_recoveries'] += 1
finally:
self.stats['active_tasks'] -= 1
self.recovery_queue.task_done()
except queue.Empty:
continue
except Exception as e:
print(f"恢复工作线程异常: {e}")
def _monitoring_worker(self):
"""监控工作线程"""
while self.running:
try:
self._perform_health_check()
time.sleep(self.config['monitoring_interval'])
except Exception as e:
print(f"监控工作线程异常: {e}")
time.sleep(60) # 出错时等待1分钟
def _statistics_worker(self):
"""统计工作线程"""
while self.running:
try:
self._generate_statistics_report()
time.sleep(3600) # 每小时报告一次
except Exception as e:
print(f"统计工作线程异常: {e}")
time.sleep(300) # 出错时等待5分钟
def _perform_health_check(self):
"""执行健康检查"""
critical_paths = self.config['critical_paths']
issues_found = []
for path in critical_paths:
if os.path.exists(path):
issues = self._check_path_health(path)
issues_found.extend(issues)
else:
# 路径不存在，创建恢复任务
task = RecoveryTask(
file_path=path,
action=RecoveryAction.RECREATE_FILE,
priority=10 # 高优先级
)
self.recovery_queue.put((-task.priority, task))
issues_found.append(f"关键路径不存在: {path}")
if issues_found:
print(f"健康检查发现 {len(issues_found)} 个问题:")
for issue in issues_found:
print(f" - {issue}")
def _check_path_health(self, path: str) -> List[str]:
"""检查路径健康状态"""
issues = []
try:
# 检查权限
if not os.access(path, os.R_OK):
issues.append(f"路径可读权限问题: {path}")
self._queue_recovery_task(
path, RecoveryAction.REPAIR_PERMISSIONS, priority=8
)
if not os.access(path, os.W_OK):
issues.append(f"路径可写权限问题: {path}")
self._queue_recovery_task(
path, RecoveryAction.REPAIR_PERMISSIONS, priority=8
)
# 如果是文件，检查完整性
if os.path.isfile(path):
integrity_issues = self._check_file_integrity(path)
issues.extend(integrity_issues)
# 如果是目录，检查目录内容
elif os.path.isdir(path):
dir_issues = self._check_directory_health(path)
issues.extend(dir_issues)
except Exception as e:
issues.append(f"检查路径 {path} 时发生异常: {e}")
return issues
def _check_file_integrity(self, filepath: str) -> List[str]:
"""检查文件完整性"""
issues = []
try:
# 检查文件大小
if os.path.getsize(filepath) == 0:
issues.append(f"文件为空: {filepath}")
self._queue_recovery_task(
filepath, RecoveryAction.RESTORE_BACKUP, priority=6
)
# 检查文件是否可读
try:
with open(filepath, 'rb') as f:
f.read(1) # 尝试读取1字节
except Exception as e:
issues.append(f"文件读取失败: {filepath} - {e}")
self._queue_recovery_task(
filepath, RecoveryAction.VERIFY_INTEGRITY, priority=7
)
except Exception as e:
issues.append(f"检查文件完整性 {filepath} 时发生异常: {e}")
return issues
def _check_directory_health(self, dirpath: str) -> List[str]:
"""检查目录健康状态"""
issues = []
try:
# 检查临时文件
for filename in os.listdir(dirpath):
if filename.startswith('.tmp') or filename.startswith('tmp'):
filepath = os.path.join(dirpath, filename)
# 检查文件年龄
file_age = time.time() - os.path.getctime(filepath)
if file_age > 86400: # 超过1天
issues.append(f"发现过期临时文件: {filepath}")
self._queue_recovery_task(
filepath, RecoveryAction.CLEANUP_TEMP, priority=3
)
except Exception as e:
issues.append(f"检查目录健康 {dirpath} 时发生异常: {e}")
return issues
def _queue_recovery_task(self, filepath: str, action: RecoveryAction, priority: int = 5):
"""将恢复任务加入队列"""
if not self.config['auto_recovery_enabled']:
return
task = RecoveryTask(
file_path=filepath,
action=action,
priority=priority,
max_retries=self.config['max_recovery_attempts']
)
self.stats['total_tasks'] += 1
self.recovery_queue.put((-priority, task)) # 使用负值实现最大优先级队列
def _execute_recovery_task(self, task: RecoveryTask) -> bool:
"""执行恢复任务"""
print(f"执行恢复任务: {task.action.value} - {task.file_path}")
try:
if task.action == RecoveryAction.REPAIR_PERMISSIONS:
return self._repair_permissions(task.file_path)
elif task.action == RecoveryAction.RECREATE_FILE:
return self._recreate_file(task.file_path)
elif task.action == RecoveryAction.RESTORE_BACKUP:
return self._restore_from_backup(task.file_path)
elif task.action == RecoveryAction.CLEANUP_TEMP:
return self._cleanup_temp_file(task.file_path)
elif task.action == RecoveryAction.VERIFY_INTEGRITY:
return self._verify_file_integrity(task.file_path)
else:
print(f"未知的恢复操作: {task.action}")
return False
except Exception as e:
print(f"执行恢复任务失败: {e}")
return False
def _repair_permissions(self, path: str) -> bool:
"""修复权限问题"""
try:
if os.path.isfile(path):
os.chmod(path, 0o644)
elif os.path.isdir(path):
os.chmod(path, 0o755)
return True
except Exception as e:
print(f"修复权限失败 {path}: {e}")
return False
def _recreate_file(self, path: str) -> bool:
"""重新创建文件"""
try:
os.makedirs(os.path.dirname(path) or '.', exist_ok=True)
with open(path, 'w') as f:
f.write('') # 创建空文件
return True
except Exception as e:
print(f"重新创建文件失败 {path}: {e}")
return False
def _restore_from_backup(self, path: str) -> bool:
"""从备份恢复文件"""
try:
# 查找最新的备份
backup_patterns = [
f"{path}.bak",
f"{path}.backup",
f"{path}.*.bak"
]
for pattern in backup_patterns:
import glob
backups = glob.glob(pattern)
if backups:
# 选择最新的备份
latest_backup = max(backups, key=os.path.getctime)
shutil.copy2(latest_backup, path)
return True
return False
except Exception as e:
print(f"从备份恢复失败 {path}: {e}")
return False
def _cleanup_temp_file(self, path: str) -> bool:
"""清理临时文件"""
try:
if os.path.exists(path):
os.remove(path)
return True
except Exception as e:
print(f"清理临时文件失败 {path}: {e}")
return False
def _verify_file_integrity(self, path: str) -> bool:
"""验证文件完整性"""
try:
if os.path.exists(path):
# 尝试完整读取文件
with open(path, 'rb') as f:
f.read()
return True
return False
except Exception as e:
print(f"验证文件完整性失败 {path}: {e}")
return False
def _generate_statistics_report(self):
"""生成统计报告"""
print(f"\n=== 自动恢复系统统计报告 ({datetime.now().strftime('%Y-%m-%d %H:%M:%S')}) ===")
print(f"总任务数: {self.stats['total_tasks']}")
print(f"成功恢复: {self.stats['successful_recoveries']}")
print(f"失败恢复: {self.stats['failed_recoveries']}")
print(f"活跃任务: {self.stats['active_tasks']}")
print(f"队列大小: {self.recovery_queue.qsize()}")
success_rate = 0
if self.stats['total_tasks'] > 0:
success_rate = (self.stats['successful_recoveries'] / self.stats['total_tasks']) * 100
print(f"成功率: {success_rate:.2f}%")
def add_recovery_task(self, filepath: str, action: RecoveryAction, priority: int = 5):
"""手动添加恢复任务"""
task = RecoveryTask(
file_path=filepath,
action=action,
priority=priority,
max_retries=self.config['max_recovery_attempts']
)
self.stats['total_tasks'] += 1
self.recovery_queue.put((-priority, task))
def shutdown(self):
"""关闭恢复系统"""
print("正在关闭自动恢复系统...")
self.running = False
# 等待所有任务完成
self.recovery_queue.join()
# 关闭线程池
self.executor.shutdown(wait=True)
print("自动恢复系统已关闭")
def demo_automated_recovery():
"""演示自动化恢复系统"""
print("=== 自动化恢复系统演示 ===")
# 创建测试目录和文件
test_dir = 'test_recovery_dir'
os.makedirs(test_dir, exist_ok=True)
# 更新配置以包含测试目录
test_config = {
'monitoring_interval': 10, # 10秒检查一次
'critical_paths': [test_dir],
'auto_recovery_enabled': True
}
# 保存测试配置
with open('test_recovery_config.json', 'w') as f:
json.dump(test_config, f)
recovery_system = AutomatedRecoverySystem('test_recovery_config.json')
try:
# 创建一些有问题的文件
print("\n创建测试问题文件...")
# 创建权限有问题的文件
problem_file = os.path.join(test_dir, 'permission_test.txt')
with open(problem_file, 'w') as f:
f.write('test')
os.chmod(problem_file, 0o000) # 移除所有权限
# 创建临时文件
temp_file = os.path.join(test_dir, '.tmp_old_file')
with open(temp_file, 'w') as f:
f.write('temporary')
# 手动添加一些恢复任务
recovery_system.add_recovery_task(
problem_file, RecoveryAction.REPAIR_PERMISSIONS, priority=9
)
recovery_system.add_recovery_task(
temp_file, RecoveryAction.CLEANUP_TEMP, priority=4
)
print("等待恢复任务执行...")
time.sleep(15) # 等待15秒让系统处理任务
# 显示统计信息
recovery_system._generate_statistics_report()
# 检查恢复结果
print("\n检查恢复结果:")
if os.path.exists(problem_file):
readable = os.access(problem_file, os.R_OK)
writable = os.access(problem_file, os.W_OK)
print(f" 权限文件恢复: 可读={readable}, 可写={writable}")
temp_exists = os.path.exists(temp_file)
print(f" 临时文件清理: 已清理={not temp_exists}")
finally:
# 清理测试文件
try:
import shutil
if os.path.exists(test_dir):
shutil.rmtree(test_dir)
if os.path.exists('test_recovery_config.json'):
os.remove('test_recovery_config.json')
except:
pass
# 关闭恢复系统
recovery_system.shutdown()
demo_automated_recovery()
---