Merge pull request #40 from FACN1/streams-and-buffers

Streams and buffers

Author: macintoshhelper

week-5/streamsAndBuffers/.gitignore

@@ -0,0 +1 @@
+node_modules

week-5/streamsAndBuffers/illiad.txt

@@ -0,0 +1,5 @@
+hello
+my
+name
+is
+this

week-5/streamsAndBuffers/mobydick.txt

@@ -0,0 +1,14 @@
+{
+  "name": "streams-and-buffers",
+  "version": "1.0.0",
+  "description": "Learn about streams",
+  "main": "readStr.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "Mavis and Jack",
+  "license": "ISC",
+  "devDependencies": {
+    "split": "^1.0.0"
+  }
+}

week-5/streamsAndBuffers/out.txt

@@ -0,0 +1,47 @@
+const fs = require('fs');
+
+//Need to require this to split by lines
+const split = require('split');
+
+// our programm can take three flags, which are then assigned below, or default with the OR operator:
+var filepath = process.argv[2]||'./test.txt'
+var hwm = process.argv[3] || 100
+var timeout = process.argv[4] || 500
+
+// Create the readable stream of the filepath specified and the hwm and set the econding to utf8
+var file = fs.createReadStream(filepath, {highWaterMark:hwm, encoding: 'utf8'});
+
+//creat empty data
+var data =''
+
+//You pipe the readable stream into split which sends it back as chunks split by a new line
+file.pipe(split())
+
+//listen for error event
+file.on('error', function(err) {
+  console.log('Error '+err);
+  return err
+});
+
+//listen for data events
+file.on('data', function(bits) {
+
+  //add new bits to the data str
+  data +=bits
+
+  //print the new bit as a string (it comes buffered)
+  process.stdout.write(bits);
+
+  //pause the reading
+  file.pause();
+
+  //set a timeout as specified by the flag (def: 500ms) to then resume the read
+  setTimeout(() => {
+    file.resume();
+  }, timeout);
+})
+
+// listen for the end event
+file.on('end', function(){
+  console.log('Finished reading all of the data');
+});

week-5/streamsAndBuffers/package.json

@@ -0,0 +1,60 @@
+# Streams & Buffers
+
+### 1) Streams
+
+#### Data Access in Node
+-  Fully buffered access vs. partially buffered access - these differ in terms of:
+  - How the data is exposed
+  - The amount of memory used to store the data
+
+#### Fully Buffered Access
+- Asynchronous `readFile()`
+- Synchronous `readFileSync()`
+- The data is returned in **one big chunk** after all of it has been read
+- Node needs to allocate enough memory to store all of that data
+
+#### Partially Buffered Access
+- Asynchronous `read()` `createReadStream()`
+- Synchronous `readSync()`
+- Data input is treated as a **series of events**, occurring as the data is being read or written - this allows us to **access data as it is being read**
+- Partially-buffered methods (listed above) allow us to specify the size of the buffer and read the data in small chunks
+
+#### Streams
+- Streams are a form of **partially buffered access**
+- Streams return small pieces of data (using a buffer) and trigger an event when each new piece of data is available for processing
+- Since streams are partially buffered, we can process each piece of data **as soon as it is read** - this means we don't necessarily need to buffer / save the whole file into memory
+- The Node stream interface consists of two parts:
+  - Readable streams
+  - Writeable streams
+
+#### Readable Streams
+- Readable streams emit the following events:
+  - `data` emits a buffer / string
+  - `end` is emitted when the stream receives an EOF (End of Function) - this indicates that there will be no more `data` events
+  - `error` is emitted if there's an error receiving data
+- To bind a callback to an event, use `stream.on(eventname, callback)` - the HTTP request object is one example of a readable stream that we've already covered:
+  - `request.on('error', function(err) { ... });`
+  - `request.on('data', function(data) { ... });`
+  - `request.on('end', function() { ... });`
+- Readable streams also have the following methods:
+  - `request.pause()` pauses incoming data events
+  - `request.resume()` resumes after a `pause()`
+  - `request.destroy()` stops the stream emitting any more events
+
+### 2) Buffers
+- In Node, buffers are a **high-performance alternative** to strings, representing raw C memory allocation
+- Buffers act like fixed-size arrays of integers, each represented as a **hexidecimal number** when run through `console.log()`
+
+```js
+var buffer = new Buffer(10);
+buffer[0] = 255;
+console.log(buffer); // <Buffer ff 00 00 00 00 4a 00 00 00 00>
+```
+- Since buffers represent raw memory, any buffer content you haven't assigned (anything other than `buffer[0]` in the above example) will be whatever happens to sit in that position in memory (e.g. the `4a` seen above)
+- Compared to strings, buffers are fixed in size, and have very limited predefined functions (missing convenient ones like `String.replace()` for example)
+- If you want to use string functions on buffers, you can convert them to strings with `buffer.toString()` first
+
+## Resources
+- http://book.mixu.net/node/ch9.html
+- https://www.sitepoint.com/basics-node-js-streams/
+- https://www.tutorialspoint.com/nodejs/nodejs_streams.htm

week-5/streamsAndBuffers/readStr.js

@@ -0,0 +1,29 @@
+this is just a test of this line repeated a lotx1
+this is just a test of this line repeated a lotx2
+this is just a test of this line repeated a lotx3
+this is just a test of this line repeated a lotx4
+this is just a test of this line repeated a lotx5
+this is just a test of this line repeated a lotx6
+this is just a test of this line repeated a lotx7
+this is just a test of this line repeated a lotx8
+this is just a test of this line repeated a lotx9
+this is just a test of this line repeated a lotx10
+this is just a test of this line repeated a lotx11
+this is just a test of this line repeated a lotx12
+this is just a test of this line repeated a lotx13
+this is just a test of this line repeated a lotx14
+this is just a test of this line repeated a lotx15
+this is just a test of this line repeated a lotx16
+this is just a test of this line repeated a lotx17
+this is just a test of this line repeated a lotx18
+this is just a test of this line repeated a lotx19
+this is just a test of this line repeated a lotx20
+this is just a test of this line repeated a lotx21
+this is just a test of this line repeated a lotx22
+this is just a test of this line repeated a lotx23
+this is just a test of this line repeated a lotx24
+this is just a test of this line repeated a lotx25
+this is just a test of this line repeated a lotx26
+this is just a test of this line repeated a lotx27
+this is just a test of this line repeated a lotx28
+this is just a test of this line repeated a lotx29

week-5/streamsAndBuffers/streams-buffers.md

@@ -0,0 +1,21 @@
+//Require the file system module
+var fs = require('fs');
+
+
+//Create a writeable stream (i.e. a stream we can write to, which will go to ./out.txt)
+var file = fs.createWriteStream('./out.txt');
+
+//Process.stdin (opposite of process.stdout) is a readable stream, that reads data that we input into the terminal
+
+//So here when we read data from process.stdin.on we write that data to our file, the writable stream we created
+process.stdin.on('data', function(data) {
+  file.write(data);
+});
+
+//When the readable stream ends, we close the writeable stream
+process.stdin.on('end', function() {
+  file.end();
+});
+
+//Resume the incoming data stream after a pause
+process.stdin.resume();