Tuesday, December 19, 2017

spark 2.2.1 installation guide on Mac

// blog install spark 2.2.1 on Mac

Spark is incompatible with Java 9.
Install Java 8 instead.

# to uninstall Java 9
brew cask uninstall java

Here is the complete and simple installation steps for spark on Mac :

brew tap caskroom/versions
brew cask search java
brew cask install java8

# brew install scala # You might need these step,
brew install apache-spark

# Start spark shell,

./bin/spark-shell --master local[2]

./bin/spark-shell --master local[1]

Scala access PostgresQL database complete demo

// Connect o PostgresQL

brew info postgres

// Start manually:

pg_ctl -D /usr/local/var/postgres start

// or, write stats to log,
pg_ctl -D /usr/local/var/postgres -l /usr/local/var/postgres/server.log start

// Stop manually:

pg_ctl -D /usr/local/var/postgres stop

// Status

pg_ctl -D /usr/local/var/postgres status

createdb scala_db
createuser scala_user
psql -U scala_user -d scala_db


create table emps
id integer,
last_name varchar(100),
email varchar(200),
gender varchar(10),
department varchar(100),
start_date date,
salary integer,
job_title varchar(100),
region_id int,
primary key (id)

insert into emps values (1,'Kelley','rkelley0@sours.getString("country")ndcloud.com','Female','Computers','10/2/2009',67470,'Structural Engineer',2);
insert into emps values (2,'Armstrong','sarmstrong1@infoseek.co.jp','Male','Home','3/31/2008',71869,'Financial Advisor',2);
insert into emps values (3,'Carr','fcarr2@woothemes.com','Male','Home','7/12/2009',101768,'Recruiting Manager',3);
insert into emps values (4,'Murray','jmurray3@gov.uk','Female','Jewelery','12/25/2014',96897,'Desktop Support Technician',3);
insert into emps values (5,'Ellis','jellis4@sciencedirect.com','Female','Home','9/19/2002',63702,'Software Engineer III',7);


// scala -classpath ~/app/lib/postgresql-42.1.1.jre6.jar
bin/spark-shell -classpath ~/app/lib/postgresql-42.1.1.jre6.jar

// Scala code,
import java.sql.Driver
import java.sql.DriverManager
import java.sql.Connection

val driver = "org.postgresql.Driver"
val url = "jdbc:postgresql://localhost/scala_db?user=scala_user"
var connection: Connection = null

connection = DriverManager.getConnection(url)
val statement = connection.createStatement()
val resultSet = statement.executeQuery("select * from emps limit 5")

while ( resultSet.next() ) {
val last_name = resultSet.getString("last_name")
val dept = resultSet.getString("department")
printf("%s: %s \n", dept, last_name)

// Prepare SQL statement
val queryStr = "select * from emps where department = ? "
val ps = connection.prepareStatement(queryStr)
ps.setString(1, "Home")
val rs = ps.executeQuery()

while ( rs.next() ) {
val last_name = rs.getString("last_name")
val dept = rs.getString("department")
val startDate = rs.getString("start_date")
printf("%s: %s \t %s \n", dept, last_name, startDate)