diff --git a/notebooks/spark_basics/DataFrame.ipynb b/notebooks/spark_basics/DataFrame.ipynb index 0108202..365e585 100644 --- a/notebooks/spark_basics/DataFrame.ipynb +++ b/notebooks/spark_basics/DataFrame.ipynb @@ -17,9 +17,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -35,6 +33,8 @@ } ], "source": [ + "from pyspark import SparkContext\n", + "sc=SparkContext.getOrCreate()\n", "# Defines a Python list storing one JSON object.\n", "json_strings = ['{\"name\":\"Bob\",\"address\":{\"city\":\"Los Angeles\",\"state\":\"California\"}}']\n", "# Defines an RDD from the Python list.\n", @@ -61,9 +61,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "data": { @@ -114,14 +112,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, diff --git a/notebooks/spark_basics/RDD.ipynb b/notebooks/spark_basics/RDD.ipynb index 0b8bf83..117857f 100644 --- a/notebooks/spark_basics/RDD.ipynb +++ b/notebooks/spark_basics/RDD.ipynb @@ -17,9 +17,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -31,6 +29,8 @@ } ], "source": [ + "from pyspark import SparkContext\n", + "sc=SparkContext.getOrCreate()\n", "text_file = sc.parallelize([\"hello\",\"hello world\"])\n", "counts = text_file.flatMap(lambda line: line.split(\" \")) \\\n", " .map(lambda word: (word, 1)) \\\n", @@ -65,14 +65,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, diff --git a/notebooks/spark_basics/spark_sql.ipynb b/notebooks/spark_basics/spark_sql.ipynb index e3f28b9..ef7aecb 100644 --- a/notebooks/spark_basics/spark_sql.ipynb +++ b/notebooks/spark_basics/spark_sql.ipynb @@ -24,9 +24,7 @@ { "cell_type": "code", "execution_count": 1, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -42,6 +40,8 @@ } ], "source": [ + "from pyspark import SparkContext\n", + "sc=SparkContext.getOrCreate()\n", "json_strings = ['{\"name\":\"Bob\",\"address\":{\"city\":\"Los Angeles\",\"state\":\"California\"}}']\n", "# Defines an RDD from the Python list.\n", "peopleRDD = sc.parallelize(json_strings)\n", @@ -60,9 +60,7 @@ { "cell_type": "code", "execution_count": 2, - "metadata": { - "collapsed": false - }, + "metadata": {}, "outputs": [ { "name": "stdout", @@ -110,14 +108,14 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.12" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, diff --git a/notebooks/spark_basics/structured_streaming.ipynb b/notebooks/spark_basics/structured_streaming.ipynb index dfc5112..957e1b6 100644 --- a/notebooks/spark_basics/structured_streaming.ipynb +++ b/notebooks/spark_basics/structured_streaming.ipynb @@ -43,7 +43,9 @@ }, "outputs": [], "source": [ - "from pyspark.streaming import StreamingContext" + "from pyspark.streaming import StreamingContext\n", + "from pyspark import SparkContext\n", + "sc=SparkContext.getOrCreate()" ] }, { @@ -227,17 +229,16 @@ "language_info": { "codemirror_mode": { "name": "ipython", - "version": 2 + "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", - "pygments_lexer": "ipython2", - "version": "2.7.10" + "pygments_lexer": "ipython3", + "version": "3.5.2" } }, "nbformat": 4, "nbformat_minor": 2 } -