Why 'shuffle' prevents trainng?

from fastai.vision.all import *

path = untar_data(URLs.MNIST_SAMPLE)
Path.BASE_PATH = path
path.ls()

(#3) [Path('train'),Path('labels.csv'),Path('valid')]

def dl(p, shuffle=False):
    def f(x):
        return [tensor(Image.open(o)) for o in (x).ls()]
    X3 = torch.stack(f(path/p/'3'))
    X7 = torch.stack(f(path/p/'7'))
    y = tensor([1]*len(X3) + [0]*len(X7))
    X = torch.cat([X3, X7]).view(-1, 28*28)/255.
    ds = list(zip(X, y))
    dl = DataLoader(ds, batch_size=256, shuffle=shuffle)
    return dl

def batch_accuracy(xb, yb):
    preds = xb.sigmoid()
    correct = (preds>0.5) == yb
    return correct.float().mean()

def mnist_loss(predictions, targets):
    predictions = predictions.sigmoid()
    return torch.where(targets==1, 1-predictions, predictions).mean()

DataLoaders without shuffle in training

This seems to be working OK as seen below.

dls = DataLoaders(dl('train'), dl('valid'))

simple_net = nn.Sequential(
    nn.Linear(28*28,30),
    nn.ReLU(),
    nn.Linear(30,1)
)
learn = Learner(dls, simple_net, opt_func=SGD, loss_func=mnist_loss, metrics=batch_accuracy)
learn.fit(5, 0.1)

plt.plot(L(learn.recorder.values).itemgot(0))
plt.plot(L(learn.recorder.values).itemgot(1))
plt.plot(L(learn.recorder.values).itemgot(2))

epoch	train_loss	valid_loss	batch_accuracy	time
0	0.337977	0.415234	0.504416	00:00
1	0.156038	0.238818	0.793164	00:00
2	0.085946	0.122527	0.906396	00:00
3	0.056482	0.086347	0.929848	00:00
4	0.042997	0.070332	0.946102	00:00

[<matplotlib.lines.Line2D at 0x7f46b7a02d00>]

DataLoaders with shuffle in training

Both losses{train, valid} and accuracy doesn't seem to be converging at all, compared with the above dls without shuffle.

dls = DataLoaders(dl('train', shuffle=True), dl('valid'))

simple_net = nn.Sequential(
    nn.Linear(28*28,30),
    nn.ReLU(),
    nn.Linear(30,1)
)
learn = Learner(dls, simple_net, opt_func=SGD, loss_func=mnist_loss, metrics=batch_accuracy)
learn.fit(5, 0.1)

plt.plot(L(learn.recorder.values).itemgot(0))
plt.plot(L(learn.recorder.values).itemgot(1))
plt.plot(L(learn.recorder.values).itemgot(2))

epoch	train_loss	valid_loss	batch_accuracy	time
0	0.499775	0.493191	0.509269	00:00
1	0.499598	0.492406	0.504416	00:00
2	0.499982	0.491580	0.504416	00:00
3	0.499273	0.491103	0.504416	00:00
4	0.499037	0.490632	0.504416	00:01

[<matplotlib.lines.Line2D at 0x7f46b6d69df0>]