Skip to content

Latest commit

 

History

History
309 lines (299 loc) · 8.05 KB

132.md

File metadata and controls

309 lines (299 loc) · 8.05 KB
Raw

Results of GPT-4 on HumanEval/132

Date: 18 July, 2023
Model: gpt-4
Temperature: 0.2
Passing runs: 0
Total runs: 10
Success rate: 0%

Prompt

Tokens: 159

def is_nested(string):
    '''
    Create a function that takes a string as input which contains only square brackets.
    The function should return True if and only if there is a valid subsequence of brackets 
    where at least one bracket in the subsequence is nested.

    is_nested('[[]]') ➞ True
    is_nested('[]]]]]]][[[[[]') ➞ False
    is_nested('[][]') ➞ False
    is_nested('[]') ➞ False
    is_nested('[[][]]') ➞ True
    is_nested('[[]][[') ➞ True
    '''

The above is an incomplete Python code fragment. Return the complete and correct code with no additional text.

Correct solution

def is_nested(string):
    '''
    Create a function that takes a string as input which contains only square brackets.
    The function should return True if and only if there is a valid subsequence of brackets 
    where at least one bracket in the subsequence is nested.

    is_nested('[[]]') ➞ True
    is_nested('[]]]]]]][[[[[]') ➞ False
    is_nested('[][]') ➞ False
    is_nested('[]') ➞ False
    is_nested('[[][]]') ➞ True
    is_nested('[[]][[') ➞ True
    '''opening_bracket_index = []
    closing_bracket_index = []
    for i in range(len(string)):
        if string[i] == '[':
            opening_bracket_index.append(i)
        else:
            closing_bracket_index.append(i)
    closing_bracket_index.reverse()
    cnt = 0
    i = 0
    l = len(closing_bracket_index)
    for idx in opening_bracket_index:
        if i < l and idx < closing_bracket_index[i]:
            cnt += 1
            i += 1
    return cnt >= 2

Unit tests

def check(candidate):

    # Check some simple cases
    assert candidate('[[]]') == True, "This prints if this assert fails 1 (good for debugging!)"
    assert candidate('[]]]]]]][[[[[]') == False
    assert candidate('[][]') == False
    assert candidate(('[]')) == False
    assert candidate('[[[[]]]]') == True
    assert candidate('[]]]]]]]]]]') == False
    assert candidate('[][][[]]') == True
    assert candidate('[[]') == False
    assert candidate('[]]') == False
    assert candidate('[[]][[') == True
    assert candidate('[[][]]') == True

    # Check some edge cases that are easy to work out by hand.
    assert candidate('') == False, "This prints if this assert fails 2 (also good for debugging!)"
    assert candidate('[[[[[[[[') == False
    assert candidate(']]]]]]]]') == False

Runs

❌ Run 1

Start time: 2023-07-18 17:58:12
Duration: 5.98s
Completion tokens: 71
Result: Fail

def is_nested(string):
    count = 0
    for i in string:
        if i == '[':
            count += 1
        elif i == ']':
            count -= 1
        if count < 0:
            return False
        if count > 1:
            return True
    return False

❌ Run 2

Start time: 2023-07-18 17:58:01
Duration: 10.78s
Completion tokens: 123
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0

def is_nested(string):
    count = 0
    for char in string:
        if char == '[':
            count += 1
        elif char == ']':
            count -= 1
        if count < 0:
            return False
    return count == 0 and '][' not in string

❌ Run 3

Start time: 2023-07-18 17:57:56
Duration: 4.98s
Completion tokens: 64
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0 and '][' not in string

❌ Run 4

Start time: 2023-07-18 17:57:48
Duration: 7.52s
Completion tokens: 64
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0 and '][' not in string

❌ Run 5

Start time: 2023-07-18 17:57:40
Duration: 8.65s
Completion tokens: 123
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0

def is_nested(string):
    count = 0
    for char in string:
        if char == '[':
            count += 1
        elif char == ']':
            count -= 1
            if count < 0:
                return False
    return count == 0 and '][' not in string

❌ Run 6

Start time: 2023-07-18 17:57:25
Duration: 14.88s
Completion tokens: 186
Result: Fail

def is_nested(string):
    '''
    Create a function that takes a string as input which contains only square brackets.
    The function should return True if and only if there is a valid subsequence of brackets 
    where at least one bracket in the subsequence is nested.

    is_nested('[[]]') ➞ True
    is_nested('[]]]]]]][[[[[]') ➞ False
    is_nested('[][]') ➞ False
    is_nested('[]') ➞ False
    is_nested('[[][]]') ➞ True
    is_nested('[[]][[') ➞ True
    '''
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0 and '][' not in string

❌ Run 7

Start time: 2023-07-18 17:57:18
Duration: 6.48s
Completion tokens: 58
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0

❌ Run 8

Start time: 2023-07-18 17:57:08
Duration: 10.37s
Completion tokens: 96
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0

def is_nested(string):
    if '[]' in string:
        return is_nested(string.replace('[]', ''))
    else:
        return '[' in string and ']' in string

❌ Run 9

Start time: 2023-07-18 17:56:59
Duration: 9.0s
Completion tokens: 96
Result: Fail

def is_nested(string):
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0

def is_nested(string):
    if '[]' in string:
        return is_nested(string.replace('[]', ''))
    else:
        return '[' in string and ']' in string

❌ Run 10

Start time: 2023-07-18 17:56:42
Duration: 16.21s
Completion tokens: 186
Result: Fail

def is_nested(string):
    '''
    Create a function that takes a string as input which contains only square brackets.
    The function should return True if and only if there is a valid subsequence of brackets 
    where at least one bracket in the subsequence is nested.

    is_nested('[[]]') ➞ True
    is_nested('[]]]]]]][[[[[]') ➞ False
    is_nested('[][]') ➞ False
    is_nested('[]') ➞ False
    is_nested('[[][]]') ➞ True
    is_nested('[[]][[') ➞ True
    '''
    stack = []
    for char in string:
        if char == '[':
            stack.append(char)
        elif char == ']':
            if not stack:
                return False
            stack.pop()
    return len(stack) == 0 and '][' not in string