Python数据分析基础
wabicai
# Python数据分析基础
# 学习目标
通过本文,你将掌握:
- Pandas数据处理的核心技能
- NumPy数值计算的关键操作
- 金融数据的常见处理方法
- 数据可视化的实用技巧
- 量化分析中的数据工程实践
# 核心库简介
# Pandas:数据处理利器
为什么用Pandas?
- 处理表格数据(股票行情、财务数据)
- 时间序列操作(K线数据、tick数据)
- 数据清洗和转换
- 统计分析和聚合
# NumPy:数值计算基础
为什么用NumPy?
- 高效的数组运算
- 矩阵操作(协方差矩阵、相关性)
- 数学函数(对数收益、标准化)
- 随机数生成(蒙特卡洛模拟)
# Matplotlib:数据可视化
为什么用Matplotlib?
- K线图、收益曲线
- 技术指标图表
- 回测结果可视化
- 数据分布分析
# Pandas核心操作
# 数据结构
import pandas as pd
import numpy as np
# Series:一维数据
prices = pd.Series([100, 102, 98, 105, 103],
index=['2024-01-01', '2024-01-02', '2024-01-03', '2024-01-04', '2024-01-05'])
print("Series:")
print(prices)
print(f"平均价格: {prices.mean():.2f}")
# DataFrame:二维表格
data = {
'open': [100, 102, 98, 105, 103],
'high': [103, 105, 101, 108, 106],
'low': [99, 101, 97, 104, 102],
'close': [102, 98, 105, 103, 107],
'volume': [1000, 1200, 800, 1500, 1100]
}
df = pd.DataFrame(data, index=pd.date_range('2024-01-01', periods=5))
print("\nDataFrame:")
print(df)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
# 数据读取与写入
import pandas as pd
# 读取CSV
df = pd.read_csv('stock_data.csv',
index_col='date', # 将date列设为索引
parse_dates=True) # 解析日期
# 读取Excel
df = pd.read_excel('portfolio.xlsx', sheet_name='持仓')
# 从API获取数据并转为DataFrame
import yfinance as yf
data = yf.download('AAPL', start='2023-01-01', end='2023-12-31')
print(data.head())
# 保存到CSV
df.to_csv('output.csv', index=True)
# 保存到Excel
df.to_excel('output.xlsx', sheet_name='数据', index=True)
# 保存到pickle(保留数据类型)
df.to_pickle('data.pkl')
df_loaded = pd.read_pickle('data.pkl')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
# 数据查看与选择
import pandas as pd
import yfinance as yf
# 下载示例数据
df = yf.download('AAPL', start='2023-01-01', end='2023-12-31')
# 查看前5行
print("前5行:")
print(df.head())
# 查看后5行
print("\n后5行:")
print(df.tail())
# 查看基本信息
print("\n数据信息:")
print(df.info())
# 查看统计信息
print("\n统计信息:")
print(df.describe())
# 选择单列
close_prices = df['Close']
# 选择多列
price_volume = df[['Close', 'Volume']]
# 使用loc(标签索引)
# 选择某一天的数据
day_data = df.loc['2023-06-01']
# 选择日期范围
range_data = df.loc['2023-06-01':'2023-06-30']
# 使用iloc(位置索引)
# 选择前10行
first_10 = df.iloc[:10]
# 选择特定行列
specific = df.iloc[0:5, 0:3] # 前5行,前3列
# 布尔索引
# 找出收盘价>150的日期
high_price_days = df[df['Close'] > 150]
# 多条件过滤
filtered = df[(df['Close'] > 150) & (df['Volume'] > 50000000)]
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
# 数据计算与转换
import pandas as pd
import numpy as np
# 下载数据
df = yf.download('AAPL', start='2023-01-01', end='2023-12-31')
# ==================== 基本计算 ====================
# 计算日收益率
df['returns'] = df['Close'].pct_change()
# 计算对数收益率
df['log_returns'] = np.log(df['Close'] / df['Close'].shift(1))
# 计算价格变化
df['price_change'] = df['Close'].diff()
# 计算百分比变化
df['pct_change'] = df['Close'].pct_change() * 100
# ==================== 移动窗口计算 ====================
# 移动平均
df['MA5'] = df['Close'].rolling(window=5).mean()
df['MA20'] = df['Close'].rolling(window=20).mean()
# 移动标准差
df['volatility'] = df['returns'].rolling(window=20).std()
# 移动最大值/最小值
df['high_20d'] = df['High'].rolling(window=20).max()
df['low_20d'] = df['Low'].rolling(window=20).min()
# 移动求和
df['volume_5d'] = df['Volume'].rolling(window=5).sum()
# ==================== 累计计算 ====================
# 累计收益
df['cumulative_returns'] = (1 + df['returns']).cumprod() - 1
# 累计最大值(用于计算回撤)
df['cum_max'] = df['Close'].cummax()
# 回撤
df['drawdown'] = (df['Close'] - df['cum_max']) / df['cum_max']
# ==================== 扩展窗口计算 ====================
# 从开始到当前的平均值
df['expanding_mean'] = df['Close'].expanding().mean()
# 从开始到当前的最大值
df['expanding_max'] = df['Close'].expanding().max()
# ==================== 时间偏移 ====================
# 向后偏移(获取昨天的值)
df['prev_close'] = df['Close'].shift(1)
# 向前偏移(获取明天的值,注意避免未来函数!)
df['next_close'] = df['Close'].shift(-1)
print("计算后的数据:")
print(df[['Close', 'returns', 'MA5', 'MA20', 'drawdown']].tail(10))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
# 数据分组与聚合
import pandas as pd
import yfinance as yf
# 下载多只股票数据
symbols = ['AAPL', 'GOOGL', 'MSFT']
data = yf.download(symbols, start='2023-01-01', end='2023-12-31')['Close']
# ==================== 分组统计 ====================
# 按月份分组
data['month'] = data.index.month
monthly_stats = data.groupby('month').agg({
'AAPL': ['mean', 'std', 'min', 'max'],
'GOOGL': ['mean', 'std'],
'MSFT': ['mean', 'std']
})
print("月度统计:")
print(monthly_stats)
# ==================== 多股票分析 ====================
# 计算收益率
returns = data.pct_change()
# 计算相关性矩阵
correlation = returns.corr()
print("\n相关性矩阵:")
print(correlation)
# 计算协方差矩阵
covariance = returns.cov()
print("\n协方差矩阵:")
print(covariance)
# ==================== 时间重采样 ====================
# 日线转周线
weekly_data = data.resample('W').agg({
'AAPL': 'last', # 收盘价取最后一个
'GOOGL': 'last',
'MSFT': 'last'
})
# 日线转月线(OHLC)
df_ohlc = df['Close'].resample('M').agg(['first', 'max', 'min', 'last'])
df_ohlc.columns = ['Open', 'High', 'Low', 'Close']
print("\n月线数据:")
print(df_ohlc)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# 数据清洗
import pandas as pd
import numpy as np
# 创建包含缺失值的示例数据
df = pd.DataFrame({
'date': pd.date_range('2023-01-01', periods=10),
'price': [100, 102, np.nan, 105, 103, np.nan, 108, 110, np.nan, 115],
'volume': [1000, 1200, 800, 0, 1500, -100, 1100, 1300, 1400, 1600]
})
print("原始数据:")
print(df)
# ==================== 处理缺失值 ====================
# 检查缺失值
print("\n缺失值统计:")
print(df.isnull().sum())
# 删除包含缺失值的行
df_dropped = df.dropna()
# 填充缺失值
df['price_filled_forward'] = df['price'].fillna(method='ffill') # 前向填充
df['price_filled_mean'] = df['price'].fillna(df['price'].mean()) # 均值填充
df['price_interpolate'] = df['price'].interpolate() # 线性插值
# ==================== 处理异常值 ====================
# 删除异常值(负数和零)
df_clean = df[df['volume'] > 0]
# 替换异常值
df['volume_clean'] = df['volume'].where(df['volume'] > 0, np.nan)
df['volume_clean'] = df['volume_clean'].fillna(method='ffill')
# 使用3σ原则识别异常值
mean = df['volume'].mean()
std = df['volume'].std()
df['is_outlier'] = (df['volume'] < mean - 3*std) | (df['volume'] > mean + 3*std)
print("\n清洗后的数据:")
print(df)
# ==================== 数据类型转换 ====================
# 转换为日期类型
df['date'] = pd.to_datetime(df['date'])
# 转换为数值类型
df['volume'] = pd.to_numeric(df['volume'], errors='coerce')
# 设置日期索引
df.set_index('date', inplace=True)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
# NumPy核心操作
# 数组创建与操作
import numpy as np
# ==================== 创建数组 ====================
# 从列表创建
arr1 = np.array([1, 2, 3, 4, 5])
print("一维数组:", arr1)
# 创建二维数组
arr2 = np.array([[1, 2, 3], [4, 5, 6]])
print("\n二维数组:\n", arr2)
# 创建特殊数组
zeros = np.zeros((3, 4)) # 全0
ones = np.ones((2, 3)) # 全1
identity = np.eye(3) # 单位矩阵
random_arr = np.random.rand(3, 3) # 随机数
# 创建序列
seq = np.arange(0, 10, 2) # [0, 2, 4, 6, 8]
linspace = np.linspace(0, 1, 5) # [0, 0.25, 0.5, 0.75, 1]
# ==================== 数组运算 ====================
a = np.array([1, 2, 3, 4])
b = np.array([5, 6, 7, 8])
# 基本运算
print("\n加法:", a + b)
print("减法:", a - b)
print("乘法:", a * b)
print("除法:", a / b)
print("幂运算:", a ** 2)
# 向量运算
dot_product = np.dot(a, b) # 点积
print("点积:", dot_product)
# 矩阵运算
A = np.array([[1, 2], [3, 4]])
B = np.array([[5, 6], [7, 8]])
matrix_mult = np.matmul(A, B) # 矩阵乘法
print("\n矩阵乘法:\n", matrix_mult)
# ==================== 统计函数 ====================
data = np.array([10, 20, 15, 25, 30, 18])
print("\n统计信息:")
print(f"均值: {np.mean(data)}")
print(f"中位数: {np.median(data)}")
print(f"标准差: {np.std(data)}")
print(f"方差: {np.var(data)}")
print(f"最大值: {np.max(data)}")
print(f"最小值: {np.min(data)}")
print(f"总和: {np.sum(data)}")
# ==================== 条件选择 ====================
arr = np.array([1, 5, 3, 8, 2, 9, 4])
# 布尔索引
greater_than_5 = arr[arr > 5]
print("\n大于5的元素:", greater_than_5)
# where条件选择
result = np.where(arr > 5, arr, 0) # 大于5保留,否则为0
print("条件选择:", result)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
# 金融计算实战
import numpy as np
import pandas as pd
# ==================== 收益率计算 ====================
prices = np.array([100, 102, 98, 105, 103, 108])
# 简单收益率
simple_returns = (prices[1:] - prices[:-1]) / prices[:-1]
print("简单收益率:", simple_returns)
# 对数收益率
log_returns = np.log(prices[1:] / prices[:-1])
print("对数收益率:", log_returns)
# ==================== 波动率计算 ====================
# 历史波动率(年化)
daily_volatility = np.std(log_returns)
annual_volatility = daily_volatility * np.sqrt(252)
print(f"\n年化波动率: {annual_volatility:.2%}")
# ==================== 夏普比率 ====================
risk_free_rate = 0.03 # 无风险利率3%
excess_returns = log_returns - risk_free_rate/252 # 超额收益
sharpe_ratio = np.mean(excess_returns) / np.std(excess_returns) * np.sqrt(252)
print(f"夏普比率: {sharpe_ratio:.2f}")
# ==================== 最大回撤 ====================
cumulative_returns = np.cumprod(1 + simple_returns) - 1
running_max = np.maximum.accumulate(cumulative_returns)
drawdown = (cumulative_returns - running_max) / (1 + running_max)
max_drawdown = np.min(drawdown)
print(f"最大回撤: {max_drawdown:.2%}")
# ==================== 相关性与协方差 ====================
# 两只股票的收益率
stock_a_returns = np.random.normal(0.001, 0.02, 100)
stock_b_returns = np.random.normal(0.0008, 0.015, 100)
# 相关系数
correlation = np.corrcoef(stock_a_returns, stock_b_returns)[0, 1]
print(f"\n相关系数: {correlation:.4f}")
# 协方差
covariance = np.cov(stock_a_returns, stock_b_returns)[0, 1]
print(f"协方差: {covariance:.6f}")
# ==================== 投资组合优化 ====================
# 三只股票的预期收益和协方差矩阵
expected_returns = np.array([0.12, 0.10, 0.08]) # 12%, 10%, 8%
cov_matrix = np.array([
[0.04, 0.01, 0.02],
[0.01, 0.03, 0.015],
[0.02, 0.015, 0.05]
])
# 等权重组合
weights = np.array([1/3, 1/3, 1/3])
# 组合预期收益
portfolio_return = np.dot(weights, expected_returns)
print(f"\n组合预期收益: {portfolio_return:.2%}")
# 组合波动率
portfolio_variance = np.dot(weights, np.dot(cov_matrix, weights))
portfolio_volatility = np.sqrt(portfolio_variance)
print(f"组合波动率: {portfolio_volatility:.2%}")
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
# 数据可视化
# 基础图表
import matplotlib.pyplot as plt
import pandas as pd
import yfinance as yf
# 设置中文字体
plt.rcParams['font.sans-serif'] = ['Arial Unicode MS']
plt.rcParams['axes.unicode_minus'] = False
# 下载数据
df = yf.download('AAPL', start='2023-01-01', end='2023-12-31')
# ==================== 折线图 ====================
plt.figure(figsize=(12, 6))
plt.plot(df.index, df['Close'], label='收盘价', linewidth=2)
plt.plot(df.index, df['Close'].rolling(20).mean(), label='20日均线', linewidth=1.5)
plt.title('AAPL 股价走势', fontsize=16)
plt.xlabel('日期')
plt.ylabel('价格 ($)')
plt.legend()
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
# ==================== 柱状图 ====================
plt.figure(figsize=(12, 6))
returns = df['Close'].pct_change()
plt.bar(df.index, returns, color=['g' if x > 0 else 'r' for x in returns], alpha=0.7)
plt.title('日收益率', fontsize=16)
plt.xlabel('日期')
plt.ylabel('收益率')
plt.axhline(y=0, color='black', linestyle='-', linewidth=0.5)
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
# ==================== 多子图 ====================
fig, axes = plt.subplots(3, 1, figsize=(12, 10))
# 子图1:价格和均线
axes[0].plot(df.index, df['Close'], label='收盘价')
axes[0].plot(df.index, df['Close'].rolling(20).mean(), label='MA20')
axes[0].set_title('价格走势')
axes[0].legend()
axes[0].grid(True, alpha=0.3)
# 子图2:成交量
axes[1].bar(df.index, df['Volume'], alpha=0.7, color='steelblue')
axes[1].set_title('成交量')
axes[1].grid(True, alpha=0.3)
# 子图3:收益率分布
axes[2].hist(returns.dropna(), bins=50, alpha=0.7, color='green')
axes[2].set_title('收益率分布')
axes[2].set_xlabel('收益率')
axes[2].set_ylabel('频数')
axes[2].grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
# 专业金融图表
import mplfinance as mpf
import pandas as pd
import yfinance as yf
# 下载数据
df = yf.download('AAPL', start='2023-10-01', end='2023-12-31')
# ==================== K线图 ====================
# 基础K线图
mpf.plot(df, type='candle', style='charles',
title='AAPL K线图',
ylabel='价格 ($)',
volume=True,
show_nontrading=False)
# ==================== K线图 + 均线 ====================
# 添加移动平均线
ma5 = df['Close'].rolling(5).mean()
ma20 = df['Close'].rolling(20).mean()
apds = [
mpf.make_addplot(ma5, color='blue', width=1),
mpf.make_addplot(ma20, color='red', width=1)
]
mpf.plot(df, type='candle', style='charles',
addplot=apds,
title='AAPL K线图 + 均线',
ylabel='价格 ($)',
volume=True,
show_nontrading=False)
# ==================== 自定义样式 ====================
mc = mpf.make_marketcolors(
up='red', # 上涨K线颜色
down='green', # 下跌K线颜色
edge='inherit',
wick='inherit',
volume='in'
)
s = mpf.make_mpf_style(marketcolors=mc, gridstyle='--', y_on_right=False)
mpf.plot(df, type='candle', style=s,
title='自定义样式K线图',
ylabel='价格 ($)',
volume=True)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
# 实战案例
# 案例1:构建技术指标库
"""
技术指标库
包含常用技术指标的计算函数
"""
import pandas as pd
import numpy as np
class TechnicalIndicators:
"""技术指标计算类"""
@staticmethod
def SMA(data, period):
"""简单移动平均"""
return data.rolling(window=period).mean()
@staticmethod
def EMA(data, period):
"""指数移动平均"""
return data.ewm(span=period, adjust=False).mean()
@staticmethod
def RSI(data, period=14):
"""相对强弱指标"""
delta = data.diff()
gain = (delta.where(delta > 0, 0)).rolling(window=period).mean()
loss = (-delta.where(delta < 0, 0)).rolling(window=period).mean()
rs = gain / loss
rsi = 100 - (100 / (1 + rs))
return rsi
@staticmethod
def MACD(data, fast=12, slow=26, signal=9):
"""MACD指标"""
ema_fast = data.ewm(span=fast, adjust=False).mean()
ema_slow = data.ewm(span=slow, adjust=False).mean()
macd_line = ema_fast - ema_slow
signal_line = macd_line.ewm(span=signal, adjust=False).mean()
histogram = macd_line - signal_line
return pd.DataFrame({
'MACD': macd_line,
'Signal': signal_line,
'Histogram': histogram
})
@staticmethod
def Bollinger_Bands(data, period=20, std_dev=2):
"""布林带"""
sma = data.rolling(window=period).mean()
std = data.rolling(window=period).std()
upper_band = sma + (std * std_dev)
lower_band = sma - (std * std_dev)
return pd.DataFrame({
'SMA': sma,
'Upper': upper_band,
'Lower': lower_band
})
@staticmethod
def ATR(high, low, close, period=14):
"""平均真实波幅"""
tr1 = high - low
tr2 = abs(high - close.shift())
tr3 = abs(low - close.shift())
tr = pd.concat([tr1, tr2, tr3], axis=1).max(axis=1)
atr = tr.rolling(window=period).mean()
return atr
# 使用示例
if __name__ == '__main__':
import yfinance as yf
# 下载数据
df = yf.download('AAPL', start='2023-01-01', end='2023-12-31')
# 计算指标
ti = TechnicalIndicators()
df['SMA_20'] = ti.SMA(df['Close'], 20)
df['EMA_12'] = ti.EMA(df['Close'], 12)
df['RSI'] = ti.RSI(df['Close'])
macd = ti.MACD(df['Close'])
df = pd.concat([df, macd], axis=1)
bb = ti.Bollinger_Bands(df['Close'])
df = pd.concat([df, bb], axis=1)
df['ATR'] = ti.ATR(df['High'], df['Low'], df['Close'])
print(df[['Close', 'SMA_20', 'RSI', 'MACD']].tail(10))
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
# 案例2:数据管道
"""
数据管道:从下载到处理的完整流程
"""
import pandas as pd
import numpy as np
import yfinance as yf
from datetime import datetime, timedelta
class DataPipeline:
"""数据处理管道"""
def __init__(self, symbol, start_date, end_date):
self.symbol = symbol
self.start_date = start_date
self.end_date = end_date
self.data = None
def download(self):
"""下载数据"""
print(f"下载 {self.symbol} 数据...")
self.data = yf.download(self.symbol, start=self.start_date, end=self.end_date)
print(f"下载完成!共 {len(self.data)} 条记录")
return self
def clean(self):
"""清洗数据"""
print("清洗数据...")
# 删除缺失值
self.data = self.data.dropna()
# 删除异常值(价格为0或负数)
self.data = self.data[self.data['Close'] > 0]
# 删除成交量异常小的日期(可能是非交易日)
self.data = self.data[self.data['Volume'] > 1000]
print(f"清洗后剩余 {len(self.data)} 条记录")
return self
def add_features(self):
"""添加特征"""
print("添加特征...")
# 收益率
self.data['returns'] = self.data['Close'].pct_change()
self.data['log_returns'] = np.log(self.data['Close'] / self.data['Close'].shift(1))
# 移动平均
for period in [5, 10, 20, 60]:
self.data[f'MA{period}'] = self.data['Close'].rolling(period).mean()
# 波动率
self.data['volatility'] = self.data['returns'].rolling(20).std() * np.sqrt(252)
# 涨跌幅
self.data['change'] = self.data['Close'] - self.data['Open']
self.data['change_pct'] = (self.data['Close'] - self.data['Open']) / self.data['Open'] * 100
# 振幅
self.data['amplitude'] = (self.data['High'] - self.data['Low']) / self.data['Low'] * 100
# 成交额
self.data['amount'] = self.data['Close'] * self.data['Volume']
print(f"添加了 {len(self.data.columns) - 6} 个特征")
return self
def add_indicators(self):
"""添加技术指标"""
print("添加技术指标...")
ti = TechnicalIndicators()
# RSI
self.data['RSI'] = ti.RSI(self.data['Close'])
# MACD
macd = ti.MACD(self.data['Close'])
self.data = pd.concat([self.data, macd], axis=1)
# 布林带
bb = ti.Bollinger_Bands(self.data['Close'])
self.data = pd.concat([self.data, bb], axis=1)
# ATR
self.data['ATR'] = ti.ATR(self.data['High'], self.data['Low'], self.data['Close'])
print("技术指标添加完成")
return self
def remove_na(self):
"""删除NA值"""
before = len(self.data)
self.data = self.data.dropna()
after = len(self.data)
print(f"删除 {before - after} 条含NA的记录")
return self
def save(self, filename):
"""保存数据"""
self.data.to_csv(filename)
print(f"数据已保存到: {filename}")
return self
def get_data(self):
"""获取处理后的数据"""
return self.data
# 使用示例
if __name__ == '__main__':
# 链式调用
pipeline = DataPipeline('AAPL', '2023-01-01', '2023-12-31')
data = (pipeline
.download()
.clean()
.add_features()
.add_indicators()
.remove_na()
.save('AAPL_processed.csv')
.get_data())
print("\n最终数据:")
print(data[['Close', 'returns', 'MA20', 'RSI', 'MACD']].tail())
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
# 常见问题
# Q1: SettingWithCopyWarning警告
问题:
df[df['Close'] > 100]['returns'] = df['Close'].pct_change()
# SettingWithCopyWarning
1
2
2
解决:
# 方法1:使用loc
df.loc[df['Close'] > 100, 'returns'] = df['Close'].pct_change()
# 方法2:使用copy()
df_filtered = df[df['Close'] > 100].copy()
df_filtered['returns'] = df_filtered['Close'].pct_change()
1
2
3
4
5
6
2
3
4
5
6
# Q2: 处理多索引DataFrame
问题:
# yfinance下载多股票返回多索引
data = yf.download(['AAPL', 'GOOGL'], start='2023-01-01')
# 列是MultiIndex: (Close, AAPL), (Close, GOOGL), ...
1
2
3
2
3
解决:
# 只保留Close列
close_prices = data['Close']
# 或者展平多索引
data.columns = ['_'.join(col).strip() for col in data.columns.values]
1
2
3
4
5
2
3
4
5
# Q3: 内存占用过大
问题: 处理大量数据时内存不足
解决:
# 1. 使用合适的数据类型
df['volume'] = df['volume'].astype('int32') # 从int64降到int32
df['price'] = df['price'].astype('float32') # 从float64降到float32
# 2. 分块处理
for chunk in pd.read_csv('large_file.csv', chunksize=10000):
process(chunk)
# 3. 只读需要的列
df = pd.read_csv('file.csv', usecols=['date', 'close', 'volume'])
# 4. 使用Dask处理超大数据
import dask.dataframe as dd
df = dd.read_csv('huge_file.csv')
1
2
3
4
5
6
7
8
9
10
11
12
13
14
2
3
4
5
6
7
8
9
10
11
12
13
14
# 进阶资源
# 推荐书籍
《利用Python进行数据分析》 - Wes McKinney
- Pandas作者亲著
- 数据分析必读
《Python for Data Analysis》 - 英文原版
- 更新更及时
- 官方案例丰富
《Python金融大数据分析》 - Yves Hilpisch
- 金融数据处理专著
# 在线资源
官方文档:
- Pandas: https://pandas.pydata.org/docs/
- NumPy: https://numpy.org/doc/
- Matplotlib: https://matplotlib.org/
教程:
- Pandas官方教程:10 Minutes to pandas
- NumPy快速入门:NumPy Quickstart
- Real Python:Python Data Analysis系列
# 练习项目
下载并分析A股数据
- 使用Tushare或AKShare
- 计算市场宽度指标
- 行业轮动分析
构建因子库
- 技术因子(动量、反转)
- 基本面因子(PE、PB、ROE)
- 另类因子(社交媒体情绪)
回测框架
- 用Pandas实现简单回测
- 性能指标计算
- 可视化报告生成
# 下一步
完成数据分析基础后,你可以:
深入学习
- 进入量化交易基础
- 学习更多策略类型
实战练习
- 分析不同行业的股票
- 寻找价格异常和套利机会
- 构建自己的数据分析工具库
扩展技能
- 学习SQL操作数据库
- 掌握数据爬虫技术
- 了解大数据处理框架(Spark、Dask)
恭喜!你已经完成了快速入门部分!
你现在掌握了:
- ✅ Python数据分析核心技能
- ✅ Pandas和NumPy的实用操作
- ✅ 金融数据处理方法
- ✅ 数据可视化技巧
准备好进入量化交易的核心知识了吗?让我们继续前进!
上一篇:第一个量化策略-双均线 | 下一篇:量化交易概述与核心概念