Merge pull request #37 from alexrudall/validate_jsonl

Add JSONL validation

Author: alexrudall

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.1] - 2021-04-11
+
+### Added
+
+- Add validation of JSONL files to make it easier to debug during upload.
+
 ## [1.2.0] - 2021-04-08
 
 ### Added

lib/ruby/openai/files.rb

@@ -18,13 +18,15 @@ def list(version: default_version)
     end
 
     def upload(version: default_version, parameters: {})
+      file = validate(file: parameters[:file])
+
       self.class.post(
         "/#{version}/files",
         headers: {
           "Content-Type" => "application/json",
           "Authorization" => "Bearer #{@access_token}"
         },
-        body: parameters.merge(file: File.open(parameters[:file]))
+        body: parameters.merge(file: file)
       )
     end
 
@@ -53,5 +55,13 @@ def delete(id:, version: default_version)
     def default_version
       "v1".freeze
     end
+
+    def validate(file:)
+      File.open(file).each_line.with_index do |line, index|
+        JSON.parse(line)
+      rescue JSON::ParserError => e
+        raise JSON::ParserError, "#{e.message} - found on line #{index + 1} of #{file}"
+      end
+    end
   end
 end

lib/ruby/openai/version.rb

@@ -1,5 +1,5 @@
 module Ruby
   module OpenAI
-    VERSION = "1.2.0".freeze
+    VERSION = "1.2.1".freeze
   end
 end

spec/fixtures/files/errors/missing_quote.jsonl

@@ -0,0 +1,2 @@
+{"text": "puppy A is happy", "metadata": "emotional state of puppy A"}
+{"text": "puppy B is sad", "metadata": "emotional state of puppy B}

spec/ruby/openai/client/files_spec.rb

@@ -11,12 +11,20 @@
         OpenAI::Client.new.files.upload(parameters: { file: file, purpose: purpose })
       end
 
-      it "succeeds" do
-        VCR.use_cassette(cassette) do
-          r = JSON.parse(response.body)
-          expect(r["filename"]).to eq(filename)
+      context "with a valid JSON lines file" do
+        it "succeeds" do
+          VCR.use_cassette(cassette) do
+            r = JSON.parse(response.body)
+            expect(r["filename"]).to eq(filename)
+          end
         end
       end
+
+      context "with an invalid file" do
+        let(:filename) { File.join("errors", "missing_quote.jsonl") }
+
+        it { expect { response }.to raise_error(JSON::ParserError) }
+      end
     end
 
     describe "#list" do