How do I download a Google Sheet with Google Picker all in JavaScript?

I'm trying to implement Google Picker and the Google Drive API in JavaScript on my website. Currently, I use a PHP script to fetch Google Drive documents, but it's using restricted scopes and I want to remove restricted scopes from my application.

First, I got the Google Picker quickstart code working. I tried to add a Google Drive get using the access token that I fetched in the Google Picker code. Google Drive code comes over in the client.js, right? Is the access token used in api.js compatible with the access token used for client.js?

I found an old Gist from six years ago and tried to integrate and update it. Here's my code right now. The gapi.client.drive.files.get fails to get the file.

// Scope to use to access user's photos.
var scope = 'https://www.googleapis.com/auth/drive.file';

var pickerApiLoaded = false;
var driveApiLoaded = false;
var oauthToken;

// Use the API Loader script to load google.picker and gapi.auth.
function onApiLoad() {
    gapi.load('auth2', onAuthApiLoad);
    gapi.load('picker', onPickerApiLoad);
}

function onClientLoad() {
    gapi.client.setApiKey(developerKey);
    gapi.client.load('drive', 'v2', onDriveApiLoad);
}

function onAuthApiLoad() {
    var authBtn = document.getElementById('auth');
    authBtn.disabled = false;
    authBtn.addEventListener('click', function() {
        gapi.auth2.init({ client_id: clientId }).then(function(googleAuth) {
            googleAuth.signIn({ scope: scope }).then(function(result) {
                handleAuthResult(result.getAuthResponse());
            })
        })
    });
}

function onPickerApiLoad() {
    pickerApiLoaded = true;
    createPicker();
}

function onDriveApiLoad() {
    driveApiLoaded = true;
}

function handleAuthResult(authResult) {
    if (authResult && !authResult.error) {
        oauthToken = authResult.access_token;
        createPicker();
    }
}

// Create and render a Picker object for picking user Photos.
function createPicker() {
    if (pickerApiLoaded && oauthToken) {
        var view = new google.picker.DocsView(google.picker.ViewId.SPREADSHEETS);
        //view.setMimeTypes("text/csv");
        //view.setMode(google.picker.DocsViewMode.LIST);
        view.setQuery(jQuery('[updateparam="name"]').val());

        var picker = new google.picker.PickerBuilder().
                //addView(google.picker.ViewId.DOCS).
                addView(view).
                setInitialView(view).
                setOAuthToken(oauthToken).
                setDeveloperKey(developerKey).
                setCallback(pickerCallback).
                build();
        picker.setVisible(true);
    }
}

// A simple callback implementation.
function pickerCallback(data) {
    if (data[google.picker.Response.ACTION] == google.picker.Action.PICKED) {
        var doc = data[google.picker.Response.DOCUMENTS][0];
        var fileId = doc[google.picker.Document.ID];
        jQuery('[updateparam="googleDriveFileId"]').val(fileId);

        //if (driveApiLoaded) {
            var request = gapi.client.drive.files.get({
                'fileId': fileId
            });
            request.execute(function(file) {
                var xhr = new XMLHttpRequest();
                xhr.open('GET', file.downloadUrl);
                xhr.setRequestHeader('Authorization', 'Bearer ' + oauthToken);
                xhr.onload = function() {
                    console.log(xhr.responseText);
                };
                xhr.onerror = function() {
                    warningMessage.displayMessage('Failed to download Google Drive document ' + fileId);
                };          
            });
        //} else {
        //  warningMessage.displayMessage('Google Drive API has not been loaded.');
        //}
    }
    // Triggers before Picker is shown
    // else {
    //  warningMessage.displayMessage('No Google Drive document selected.');
    //}
}

And my script tags:

<!-- The Google API Loader script. -->
<script type="text/javascript" src="https://apis.google.com/js/api.js?onload=onApiLoad"></script>
<script type="text/javascript" src="https://www.google.com/jsapi?key=KEY"></script>
<script type="text/javascript" src="https://apis.google.com/js/client.js?onload=onClientLoad"></script>