- Notifications
You must be signed in to change notification settings - Fork940
Conform list tools#90
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Merged
Uh oh!
There was an error while loading.Please reload this page.
Merged
Changes fromall commits
Commits
Show all changes
2 commits Select commitHold shift + click to select a range
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -6,6 +6,7 @@ import ( | ||
| "bufio" | ||
| "context" | ||
| "encoding/json" | ||
| "errors" | ||
| "fmt" | ||
| "io" | ||
| "os" | ||
| @@ -17,6 +18,8 @@ import ( | ||
| "github.com/docker/docker/api/types/network" | ||
| "github.com/docker/docker/client" | ||
| "github.com/docker/docker/pkg/stdcopy" | ||
| "github.com/google/go-cmp/cmp" | ||
| "github.com/google/go-cmp/cmp/cmpopts" | ||
| "github.com/stretchr/testify/require" | ||
| ) | ||
| @@ -130,16 +133,19 @@ func TestCapabilities(t *testing.T) { | ||
| anthropicServer := start(t, anthropic) | ||
| githubServer := start(t, github) | ||
| req := initializeRequest{ | ||
| JSONRPC: "2.0", | ||
| ID: 1, | ||
| Method: "initialize", | ||
| Params: initializeParams{ | ||
| ProtocolVersion: "2025-03-26", | ||
| Capabilities: clientCapabilities{}, | ||
| ClientInfo: clientInfo{ | ||
| Name: "ConformanceTest", | ||
| Version: "0.0.1", | ||
| }, | ||
| }, | ||
| } | ||
| require.NoError(t, anthropicServer.send(req)) | ||
| @@ -227,6 +233,69 @@ func diffNonNilFields(a, b interface{}, path string) string { | ||
| return sb.String() | ||
| } | ||
| func TestListTools(t *testing.T) { | ||
| anthropicServer := start(t, anthropic) | ||
| githubServer := start(t, github) | ||
| req := listToolsRequest{ | ||
| JSONRPC: "2.0", | ||
| ID: 1, | ||
| Method: "tools/list", | ||
| } | ||
| require.NoError(t, anthropicServer.send(req)) | ||
| var anthropicListToolsResponse listToolsResponse | ||
| require.NoError(t, anthropicServer.receive(&anthropicListToolsResponse)) | ||
| require.NoError(t, githubServer.send(req)) | ||
| var ghListToolsResponse listToolsResponse | ||
| require.NoError(t, githubServer.receive(&ghListToolsResponse)) | ||
| require.NoError(t, isToolListSubset(anthropicListToolsResponse.Result, ghListToolsResponse.Result), "expected the github list tools response to be a subset of the anthropic list tools response") | ||
| } | ||
| func isToolListSubset(subset, superset listToolsResult) error { | ||
| // Build a map from tool name to Tool from the superset | ||
| supersetMap := make(map[string]tool) | ||
| for _, tool := range superset.Tools { | ||
| supersetMap[tool.Name] = tool | ||
| } | ||
| var err error | ||
| for _, tool := range subset.Tools { | ||
| sup, ok := supersetMap[tool.Name] | ||
| if !ok { | ||
| return fmt.Errorf("tool %q not found in superset", tool.Name) | ||
| } | ||
| // Intentionally ignore the description fields because there are lots of slight differences. | ||
| // if tool.Description != sup.Description { | ||
| // return fmt.Errorf("description mismatch for tool %q, got %q expected %q", tool.Name, tool.Description, sup.Description) | ||
| // } | ||
| // Ignore any description fields within the input schema properties for the same reason | ||
| ignoreDescOpt := cmp.FilterPath(func(p cmp.Path) bool { | ||
| // Look for a field named "Properties" somewhere in the path | ||
| for _, ps := range p { | ||
| if sf, ok := ps.(cmp.StructField); ok && sf.Name() == "Properties" { | ||
| return true | ||
| } | ||
| } | ||
| return false | ||
| }, cmpopts.IgnoreMapEntries(func(k string, _ any) bool { | ||
| return k == "description" | ||
| })) | ||
| if diff := cmp.Diff(tool.InputSchema, sup.InputSchema, ignoreDescOpt); diff != "" { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others.Learn more. The current subset check using cmp.Diff requires an exact match between input schemas. To correctly enforce that the GitHub response is a subset of the Anthropic response, consider modifying this comparison to validate that every key-value pair in the GitHub schema exists in the Anthropic schema, allowing additional properties in the latter. Copilot uses AI. Check for mistakes. | ||
| err = errors.Join(err, fmt.Errorf("inputSchema mismatch for tool %q:\n%s", tool.Name, diff)) | ||
| } | ||
| } | ||
| return err | ||
| } | ||
| type serverStartResult struct { | ||
| server server | ||
| err error | ||
| @@ -274,48 +343,39 @@ func (s server) receive(res response) error { | ||
| return res.unmarshal(line) | ||
| } | ||
| type request interface { | ||
| marshal() ([]byte, error) | ||
| } | ||
| type response interface { | ||
| unmarshal([]byte) error | ||
| } | ||
| type jsonRPRCRequest[params any] struct { | ||
| JSONRPC string `json:"jsonrpc"` | ||
| ID int `json:"id"` | ||
| Method string `json:"method"` | ||
| Params params `json:"params"` | ||
| } | ||
| func (r jsonRPRCRequest[any]) marshal() ([]byte, error) { | ||
| return json.Marshal(r) | ||
| } | ||
| type jsonRPRCResponse[result any] struct { | ||
| JSONRPC string `json:"jsonrpc"` | ||
| ID int `json:"id"` | ||
| Method string `json:"method"` | ||
| Result result `json:"result"` | ||
| } | ||
| func (r *jsonRPRCResponse[any]) unmarshal(b []byte) error { | ||
| return json.Unmarshal(b, r) | ||
| } | ||
| type initializeRequest = jsonRPRCRequest[initializeParams] | ||
| typeinitializeParams struct { | ||
| ProtocolVersion string `json:"protocolVersion"` | ||
| Capabilities clientCapabilities `json:"capabilities"` | ||
| ClientInfo clientInfo `json:"clientInfo"` | ||
| @@ -328,13 +388,7 @@ type clientInfo struct { | ||
| Version string `json:"version"` | ||
| } | ||
| type initializeResponse = jsonRPRCResponse[initializeResult] | ||
| type initializeResult struct { | ||
| ProtocolVersion string `json:"protocolVersion"` | ||
| @@ -360,3 +414,22 @@ type serverInfo struct { | ||
| Name string `json:"name"` | ||
| Version string `json:"version"` | ||
| } | ||
| type listToolsRequest = jsonRPRCRequest[struct{}] | ||
| type listToolsResponse = jsonRPRCResponse[listToolsResult] | ||
| type listToolsResult struct { | ||
| Tools []tool `json:"tools"` | ||
| } | ||
| type tool struct { | ||
| Name string `json:"name"` | ||
| Description string `json:"description,omitempty"` | ||
| InputSchema inputSchema `json:"inputSchema"` | ||
| } | ||
| type inputSchema struct { | ||
| Type string `json:"type"` | ||
| Properties map[string]any `json:"properties,omitempty"` | ||
| Required []string `json:"required,omitempty"` | ||
| } | ||
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.