# drop missing values and convert to lowercase

data = data.dropna() # Remove rows with missing data

data = data.applymap(lambda s: s.lower() if type(s) == str else s)

df1.union(df2)

df1.unionByName(df2)

df1.intersect(df2)

df1.except(df2)

df1.subtract(df2)

df.sort("column")

df.select("*", col("struct_column.*"))

df.select("*", col("nested_struct_column.level1.*"), col("nested_struct_column.level2.*"))

df.select(array_remove(col("array_column"), "value"))

df.select(array_repeat("value", 3))

df.select(size(col("array_column")))

df.select(array_sort(col("array_column")))

df.select(array_union(col("array_column1"), col("array_column2")))

df.select(arrays_zip(col("array_column1"), col("array_column2")))

df.filter(col("map_column").getItem("key").isNotNull())

df.select(map_keys(col("map_column")))

df.select(map_values(col("map_column")))

df.select(map_from_entries(col("array_column")))

df.select(map_concat(col("map_column1"), col("map_column2")))

df.select(map_zip_with(col("map_column1"), col("map_column2"), (k, v1, v2) => v1 + v2))

Plot from CSV File/DataFrame

page4211, page4735, page4278, page4500.

import pandas as pd

# Create a Series from a list
data = [10, 20, 30, 40, 50]
s = pd.Series(data)

print(s)

import pandas as pd

# Create a Series from a list
data = [10, 20, 30, 40, 50]
s = pd.Series(data, index=["a", "b", "c", "d", "e"])

print(s)

df.dtypes.unique() returns an array of the unique data types present in the DataFrame.

int64: 64-bit integers.
float64: 64-bit floating-point numbers.
O (object): columns with mixed types or strings.
bool: boolean values. 

num = ["int64", "float64"]: This line defines a list num that contains the data types int64 and float64. These represent 64-bit integers and 64-bit floating-point numbers, respectively.

num_vars = list(df.select_dtypes(include=num)):
df.select_dtypes(include=num) selects columns in the DataFrame df that have data types specified in the num list. This will include columns with data types int64 and float64.
list(...) converts the resulting selection of column names into a list.
num_vars will now be a list containing the names of columns in df that are either of type int64 or float64.

df_liao = df[num_vars]:
This creates a new DataFrame df_liao that contains only the columns from df whose names are in the num_vars list.
df_liao will be a subset of df with only the numeric columns.

import pandas as pd
data = {
'Column1': [1, 2, 3, 4],
'Column2': [5, 6, 7, 8],
'Column3': [9, 10, 11, 12],
'Column4': [13, 14, 15, 16]
}
df = pd.DataFrame(data)
print(df)

print(df.loc[0, "Column1"])
print(df.loc[1, "Column1"])
print(df.loc[0, "Column2"])

1
2
5

import pandas as pd
data = {
'Column1': [1, 2, 3, 4],
'Column2': [5, 6, 7, 8],
'Column3': [9, 10, 11, 12],
'Column4': [13, 14, 15, 16]
}
df = pd.DataFrame(data)
df.index=["a", "b", "c", "d"]
print(df)

print(df.loc["a", "Column1"])
print(df.loc["b", "Column1"])
print(df.loc["a", "Column2"])

import pandas as pd
data = {
'Column1': [1, 2, 3, 4],
'Column2': [5, 6, 7, 8],
'Column3': [9, 10, 11, 12],
'Column4': [13, 14, 15, 16]
}
df = pd.DataFrame(data)
df.index=["a", "b", "c", "d"]
print(df)

print()
z = df.iloc[0:1, 0:2]
print(z)

import pandas as pd
data = {
'Column1': [1, 2, 3, 4],
'Column2': [5, 6, 7, 8],
'Column3': [9, 10, 11, 12],
'Column4': [13, 14, 15, 16]
}
df = pd.DataFrame(data)
df.index=["a", "b", "c", "d"]
print(df)

print()
z = df.loc["a":"b", "Column2":"Column4"]
print(z)

import pandas as pd
data = {
'Column1': [1, 2, 3, 4],
'Column2': [5, 6, 7, 8],
'Column3': [9, 10, 11, 12],
'Column4': [13, 14, 15, 16]
}
df = pd.DataFrame(data)
df.index=["a", "b", "c", "d"]
print(df)

print()
z = df.iloc[0:2, 1:3] # Here 0:2 selects rows at index positions 0 and 1, and 1:3 selects columns at index positions 1 and 2
print(z)

The error indicates that PCA detected NaN values in the input data, even though the CSV appears to have no missing values. This can sometimes occur due to issues in the data processing steps before PCA.

The script does: Check for NaNs After Reading Data: Sometimes, if data is incorrectly formatted or there are hidden characters, pandas might interpret values as NaNs. Check if any NaN values are being introduced when reading the CSV file.
Verify Data Conversion and Handling: Ensure that all operations on the DataFrame, like setting indices or calculating means, are executed correctly without introducing NaNs.
Check Scaling Step: The StandardScaler might be encountering issues with the input format or specific data types.