No reason for so many loops:

private void Update()
 {
     float minDistance = float.MaxValue;
     for (int i = 0; i < characters.Length; i++)
     {
         distance[i] = Vector3.Distance(characters[i].transform.position,  center.transform.position);
         if(distance[i] < minDistance)
         {
             minDistance = distance[i];
             minCharNum = i;
         }
     }

     if (!isDragging)
     {
         LerpToChar(minCharNum * distanceBetweenCharacters);
     }
 }

I would also suggest moving this outside of the Update method, if you don't actually need this every frame and if you have a large number of characters.

No reason for so many loops:

private void Update()
 {
     float minDistance = float.MaxValue;
     for (int i = 0; i < characters.Length; i++)
     {
         distance[i] = (characters[i].transform.position - center.transform.position).sqrMagnitude;
         if(distance[i] < minDistance)
         {
             minDistance = distance[i];
             minCharNum = i;
         }
     }

     if (!isDragging)
     {
         LerpToChar(minCharNum * distanceBetweenCharacters);
     }
 }

I would also suggest moving this outside of the Update method, if you don't actually need this every frame and if you have a large number of characters.