本文通过实际案例深入探讨Python环境下数据抓取、API交互、数据读取、本地数据分析及机器学习应用，涵盖从简单HTTP请求到复杂数据处理与模型训练的全过程。
http://blog.youkuaiyun.com/pipisorry/article/details/44833603
在[1]:
%matplotlib inline
抓取的数据
一个简单的HTTP请求
在[2]:
import requests
print requests.get("http://example.com").text
与api交流
在[3]:
response = requests.get("https://www.googleapis.com/books/v1/volumes", params={"q":"machine learning"})
raw_data = response.json()
titles = [item['volumeInfo']['title'] for item in raw_data['items']]
titles
[3]:
在[4]:
import lxml.html
page = lxml.html.parse("http://www.blocket.se/stockholm?q=apple")
# ^ This is probably illegal. Blocket, please don't sue me!
items_data = []
for el in page.getroot().find_class("item_row"):
links = el.find_class("item_link")
images = el.find_class("item_image")
prices = el.find_class("list_price")
if links and images and prices and prices[0].text:
items_data.append({"name": links[0].text,
"image": images[0].attrib['src'],
"price": int(prices[0].text.split(":")[0].replace(" ", ""))})
items_data
[4]:
阅读本地数据
在[5]:
import pandas
df = pandas.read_csv('sample.csv')
在[6]:
# Display the DataFrame
df
[6]:
在[7]:
# DataFrame's columns
df.columns
[7]:
在[8]:
# Values of a given column
df.Model
[8]。
分析了dataframe
在[9]:
# Any missing values?
df['Price']
[9]:
在[10]:
df['Description']
[10]。
在[11]:
# Fill missing prices by a linear interpolation
df['Description'] = df['Description'].fillna("No description is available.")
df['Price'] = df['Price'].interpolate()
df
[11]。
探索数据
在[12]:
import matplotlib.pyplot as plt
df = pandas.read_csv('sample2.csv')
df
[12]。
在[13]:
# This table has 3 columns: Office, Year, Sales
print df.columns
# It's really easy to query data with Pandas:
print df[(df['Office'] == 'Stockholm') & (df['Sales'] > 260)]
# It's also easy to do aggregations...
aggregated_sales = df.groupby('Year').sum()
print aggregated_sales
在[14]:
# ... and generate plots
%matplotlib inline
aggregated_sales.plot(kind='bar')
[14]。
机器学习
特征提取
在[15]:
from sklearn import feature_extraction
从文本中提取特征
在[16]:
corpus = ['All the cats really are great.',
'I like the cats but I still prefer the dogs.',
'Dogs are the best.',
'I like all the trains',
]
tfidf = feature_extraction.text.TfidfVectorizer()
print tfidf.fit_transform(corpus).toarray()
print tfidf.get_feature_names()
Dict vectorizer
在[17]:
import json
data = [json.loads("""{"weight": 194.0, "sex": "female", "student": true}"""),
{"weight": 60., "sex": 'female', "student": True},
{"weight": 80.1, "sex": 'male', "student": False},
{"weight": 65.3, "sex": 'male', "student": True},
{"weight": 58.5, "sex": 'female', "student": False}]
vectorizer = feature_extraction.DictVectorizer(sparse=False)
vectors = vectorizer.fit_transform(data)
print vectors
print vectorizer.get_feature_names()
在[18]:
class A:
def __init__(self, x):
self.x = x
self.blabla = 'test'
a = A(20)
a.__dict__
出[18]:
预处理
扩展
在[19]:
from sklearn import preprocessing
data = [[10., 2345., 0., 2.],
[3., -3490., 0.1, 1.99],
[13., 3903., -0.2, 2.11]]
print preprocessing.normalize(data)
降维
在[20]:
from sklearn import decomposition
data = [[0.3, 0.2, 0.4, 0.32],
[0.3, 0.5, 1.0, 0.19],
[0.3, -0.4, -0.8, 0.22]]
pca = decomposition.PCA()
print pca.fit_transform(data)
print pca.explained_variance_ratio_
机器学习模型
分类(支持向量机)
在[21]:
from sklearn import datasets
from sklearn import svm
在[22]:
iris = datasets.load_iris()
X = iris.data[:, :2]
y = iris.target
# Training the model
clf = svm.SVC(kernel='rbf')
clf.fit(X, y)
# Doing predictions
new_data = [[4.85, 3.1], [5.61, 3.02]]
print clf.predict(new_data)
回归(线性回归)
在[23]:
import numpy as np
from sklearn import linear_model
import matplotlib.pyplot as plt
def f(x):
return x + np.random.random() * 3.
X = np.arange(0, 5, 0.5)
X = X.reshape((len(X), 1))
y = map(f, X)
clf = linear_model.LinearRegression()
clf.fit(X, y)
(23):
在[24]:
new_X = np.arange(0.2, 5.2, 0.3)
new_X = new_X.reshape((len(new_X), 1))
new_y = clf.predict(new_X)
plt.scatter(X, y, color='g', label='Training data')
plt.plot(new_X, new_y, '.-', label='Predicted')
plt.legend()
(24):
集群(DBScan)
在[25]:
from sklearn.cluster import DBSCAN
from sklearn.datasets.samples_generator import make_blobs
from sklearn.preprocessing import StandardScaler
# Generate sample data
centers = [[1, 1], [-1, -1], [1, -1]]
X, labels_true = make_blobs(n_samples=200, centers=centers, cluster_std=0.4,
random_state=0)
X = StandardScaler().fit_transform(X)
在[26]:
# Compute DBSCAN
db = DBSCAN(eps=0.3, min_samples=10).fit(X)
db.labels_
出[26]:
在[27]:
import matplotlib.pyplot as plt
plt.scatter(X[:, 0], X[:, 1], c=db.labels_)
(27):
交叉验证
在[28]:
from sklearn import svm, cross_validation, datasets
iris = datasets.load_iris()
X, y = iris.data, iris.target
model = svm.SVC()
print cross_validation.cross_val_score(model, X, y, scoring='precision')
print cross_validation.cross_val_score(model, X, y, scoring='mean_squared_error')
ref:Data-processing and machine learning with Python
扫一扫
被折叠的 条评论
为什么被折叠?
到【灌水乐园】发言
