Pandas Time Series Basics Import modules from datetime import datetime import pandas as pd %matplotlib inline import matplotlib.pyplot as pyplot Create a dataframe data = {‘date’: [‘2014-05-01 18:47:05.069722’, ‘2014-05-01 18:47:05.119994’, ‘2014-05-02 18:47:05.178768’, ‘2014-05-02 18:47:05.230071’, ‘2014-05-02 18:47:05.230071’, ‘2014-05-02 18:47:05.280592’, ‘2014-05-03 18:47:05.332662’, ‘2014-05-03 18:47:05.385109’, ‘2014-05-04 18:47:05.436523’, ‘2014-05-04 18:47:05.486877’], ‘battle_deaths’: [34, 25, 26, 15, 15, 14, 26, 25, …
Group A Time Series With pandas Import required modules import pandas as pd import numpy as np Create a dataframe df = pd.DataFrame() df[‘german_army’] = np.random.randint(low=20000, high=30000, size=100) df[‘allied_army’] = np.random.randint(low=20000, high=40000, size=100) df.index = pd.date_range(‘1/1/2014′, periods=100, freq=’H’) df.head() german_army allied_army 2014-01-01 00:00:00 21413 37604 2014-01-01 01:00:00 25913 21144 2014-01-01 02:00:00 22418 34201 2014-01-01 03:00:00 …
Convert A Variable To A Time Variable In pandas /* Import Preliminaries */ import pandas as pd /* Create a dataset with the index being a set of names */ raw_data = {‘date’: [‘2014-06-01T01:21:38.004053’, ‘2014-06-02T01:21:38.004053’, ‘2014-06-03T01:21:38.004053’], ‘score’: [25, 94, 57]} df = pd.DataFrame(raw_data, columns = [‘date’, ‘score’]) df date score 0 2014-06-01T01:21:38.004053 25 1 2014-06-02T01:21:38.004053 …
Converting Strings To Datetime Import modules from datetime import datetime from dateutil.parser import parse import pandas as pd Create a string variable with the war start time war_start = ‘2011-01-03’ Convert the string to datetime format datetime.strptime(war_start, ‘%Y-%m-%d’) datetime.datetime(2011, 1, 3, 0, 0) Create a list of strings as dates attack_dates = [‘7/2/2011’, ‘8/6/2012′, ’11/13/2013’, …
Dimensionality Reduction With PCA Preliminaries /* Load libraries */ from sklearn.preprocessing import StandardScaler from sklearn.decomposition import PCA from sklearn import datasets Load Data /* Load the data */ digits = datasets.load_digits() Standardize Feature Values /* Standardize the feature matrix */ X = StandardScaler().fit_transform(digits.data) Conduct Principal Component Analysis /* Create a PCA that will retain 99% …
Dimensionality Reduction With Kernel PCA Preliminaries /* Load libraries */ from sklearn.decomposition import PCA, KernelPCA from sklearn.datasets import make_circles Create Linearly Inseparable Data /* Create linearly inseparable data */ X, _ = make_circles(n_samples=1000, random_state=1, noise=0.1, factor=0.1) Conduct Kernel PCA /* Apply kernal PCA with radius basis function (RBF) kernel */ kpca = KernelPCA(kernel=”rbf”, gamma=15, n_components=1) …
Dimensionality Reduction On Sparse Feature Matrix Preliminaries /* Load libraries */ from sklearn.preprocessing import StandardScaler from sklearn.decomposition import TruncatedSVD from scipy.sparse import csr_matrix from sklearn import datasets import numpy as np Load Digits Data And Make Sparse /* Load the data */ digits = datasets.load_digits() /* Standardize the feature matrix */ X = StandardScaler().fit_transform(digits.data) /* …
Select Date And Time Ranges Preliminaries /* Load library */ import pandas as pd Create pandas Series Time Data /* Create data frame */ df = pd.DataFrame() /* Create datetimes */ df[‘date’] = pd.date_range(‘1/1/2001′, periods=100000, freq=’H’) Select Time Range (Method 1) Use this method if your data frame is not indexed by time. /* Select …
Rolling Time Window Preliminaries import pandas as pd Create Date Data time_index = pd.date_range(’01/01/2010′, periods=5, freq=’M’) df = pd.DataFrame(index=time_index) df[‘Stock_Price’] = [1,2,3,4,5] Create A Rolling Time Window Of Two Rows df.rolling(window=2).mean() Stock_Price 2010-01-31 NaN 2010-02-28 1.5 2010-03-31 2.5 2010-04-30 3.5 2010-05-31 4.5 /* Identify max value in rolling time window */ df.rolling(window=2).max() Stock_Price 2010-01-31 NaN …
Lag A Time Feature Preliminaries import pandas as pd Create Date Data df = pd.DataFrame() df[‘dates’] = pd.date_range(‘1/1/2001′, periods=5, freq=’D’) df[‘stock_price’] = [1.1,2.2,3.3,4.4,5.5] Lag Time Data By One Row df[‘previous_days_stock_price’] = df[‘stock_price’].shift(1) df dates stock_price previous_days_stock_price 0 2001-01-01 1.1 NaN 1 2001-01-02 2.2 1.1 2 2001-01-03 3.3 2.2 3 2001-01-04 4.4 3.3 4 2001-01-05 5.5 …